From 25b91deba171b072b3091a496f7b33331a594a2e Mon Sep 17 00:00:00 2001 From: superjom Date: Wed, 22 Nov 2017 17:08:54 +0800 Subject: [PATCH] set num_records to total_records --- CMakeLists.txt | 2 - visualdl/backend/logic/im.cc | 15 ++- visualdl/backend/storage/storage.cc | 10 +- visualdl/backend/storage/storage.proto | 164 +++++++++++++------------ 4 files changed, 100 insertions(+), 91 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04937a28..b3eec5e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,6 @@ link_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/lib) add_library(storage ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.cc ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.pb.cc) -add_library(c_api ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/c_api.cc) add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/sdk.cc) add_library(im ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im.cc) @@ -29,4 +28,3 @@ add_executable(vl_test ${PROJECT_SOURCE_DIR}/visualdl/backend/test.cc ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im_test.cc) target_link_libraries(vl_test storage im gtest glog protobuf gflags) - diff --git a/visualdl/backend/logic/im.cc b/visualdl/backend/logic/im.cc index e53f3a24..12fa5cd5 100644 --- a/visualdl/backend/logic/im.cc +++ b/visualdl/backend/logic/im.cc @@ -1,5 +1,5 @@ -#include #include +#include #include "visualdl/backend/logic/im.h" @@ -45,15 +45,14 @@ void InformationMaintainer::AddRecord(const std::string &tag, auto *tablet = storage_.Find(tag); CHECK(tablet); - auto num_records = tablet->num_records(); + auto num_records = tablet->total_records(); const auto num_samples = tablet->num_samples(); int offset; // use reservoir sampling or not if (num_samples > 0) { offset = ReserviorSample(num_samples, num_records + 1); - if (offset < 0) - return; + if (offset < 0) return; } else { offset = num_records; } @@ -66,11 +65,11 @@ void InformationMaintainer::AddRecord(const std::string &tag, } *record = data; - tablet->set_num_records(num_records + 1); + tablet->set_total_records(num_records + 1); } void InformationMaintainer::Clear() { - auto* data = storage().mutable_data(); + auto *data = storage().mutable_data(); data->clear_tablets(); data->clear_dir(); data->clear_timestamp(); @@ -79,8 +78,8 @@ void InformationMaintainer::Clear() { void InformationMaintainer::PersistToDisk() { CHECK(!storage_.data().dir().empty()) << "path of storage should be set"; // TODO make dir first - //MakeDir(storage_.data().dir()); + // MakeDir(storage_.data().dir()); storage_.Save(storage_.data().dir() + "/storage.pb"); } -} // namespace visualdl +} // namespace visualdl diff --git a/visualdl/backend/storage/storage.cc b/visualdl/backend/storage/storage.cc index 5a89aa49..7a27448c 100644 --- a/visualdl/backend/storage/storage.cc +++ b/visualdl/backend/storage/storage.cc @@ -1,5 +1,5 @@ -#include #include +#include #include "visualdl/backend/storage/storage.h" @@ -24,15 +24,15 @@ storage::Record *Storage::NewRecord(const std::string &tag) { CHECK(tablet) << "Tablet" << tag << " should be create first"; auto *record = tablet->mutable_records()->Add(); // increase num_records - int num_records = tablet->num_records(); - tablet->set_num_records(num_records + 1); + int num_records = tablet->total_records(); + tablet->set_total_records(num_records + 1); return record; } storage::Record *Storage::GetRecord(const std::string &tag, int offset) { auto *tablet = Find(tag); CHECK(tablet) << "Tablet" << tag << " should be create first"; - auto num_records = tablet->num_records(); + auto num_records = tablet->total_records(); CHECK_LT(offset, num_records) << "invalid offset"; return tablet->mutable_records()->Mutable(offset); } @@ -60,4 +60,4 @@ void Storage::DeSerialize(const std::string &data) { proto_.ParseFromString(data); } -} // namespace visualdl +} // namespace visualdl diff --git a/visualdl/backend/storage/storage.proto b/visualdl/backend/storage/storage.proto index 9c0f9d5c..a38cb3ce 100644 --- a/visualdl/backend/storage/storage.proto +++ b/visualdl/backend/storage/storage.proto @@ -2,47 +2,48 @@ syntax = "proto3"; package storage; enum DataType { - // single entry - kInt32 = 0; - kInt64 = 1; - kFloat = 2; - kDouble = 3; - kString = 4; - kBool = 5; - // entrys - kInt64s = 6; - kFloats = 7; - kDoubles = 8; - kStrings = 9; - kInt32s = 10; - kBools = 11; + // single entry + kInt32 = 0; + kInt64 = 1; + kFloat = 2; + kDouble = 3; + kString = 4; + kBool = 5; + // entrys + kInt64s = 6; + kFloats = 7; + kDoubles = 8; + kStrings = 9; + kInt32s = 10; + kBools = 11; - kUnknown = 12; + kUnknown = 12; } // A data array, which type is `type`. message Entry { - // if all the entries in a record share the same data type, ignore this field - // and store type to `dtype` in `Record`. - DataType dtype = 1; - // single element - int32 i32 = 2; - int64 i64 = 3; - string s = 4; - float f = 5; - double d = 6; - bool b = 7; - // array - repeated int64 i64s = 8; - repeated float fs = 9; - repeated double ds = 10; - repeated int32 i32s = 11; - repeated string ss = 12; - repeated bool bs = 13; + // if all the entries in a record share the same data type, ignore this field + // and store type to `dtype` in `Record`. + DataType dtype = 1; + // single element + int32 i32 = 2; + int64 i64 = 3; + string s = 4; + float f = 5; + double d = 6; + bool b = 7; + // array + repeated int64 i64s = 8; + repeated float fs = 9; + repeated double ds = 10; + repeated int32 i32s = 11; + repeated string ss = 12; + repeated bool bs = 13; } /* -The Record proto is designed to represent any data structure for any component, for +The Record proto is designed to represent any data structure for any component, +for example, to store a record of Scalar component Record { @@ -63,62 +64,73 @@ Record { meta = [100, 200] } -for other complex structure which have more fields than `timestamp`, `id` and meta, the additional fields -can store in the `data` field, for it can store a list of values in different basic data types. +for other complex structure which have more fields than `timestamp`, `id` and +meta, the additional fields +can store in the `data` field, for it can store a list of values in different +basic data types. -A component handlers in logic layer should know how to write or load a record for the corresponding component. +A component handlers in logic layer should know how to write or load a record +for the corresponding component. */ message Record { - // one record might have multiple fields, one specific component should know how - // to parse the records. - repeated Entry data = 1; - // NOTE the timestamp, id, dtype might be useless for that all the meta info can - // be stored in `data` field. - int64 timestamp = 2; - // store the count of writing operations to the tablet. - int64 id = 3; - DataType dtype = 4; - // shape or some other meta infomation for this record, if all the records - // share the same meta, just store one copy of meta in `Storage`, or create - // a unique copy for each `Record`. - Entry meta = 5; + // one record might have multiple fields, one specific component should know + // how + // to parse the records. + repeated Entry data = 1; + // NOTE the timestamp, id, dtype might be useless for that all the meta info + // can + // be stored in `data` field. + int64 timestamp = 2; + int64 id = 3; + DataType dtype = 4; + // shape or some other meta infomation for this record, if all the records + // share the same meta, just store one copy of meta in `Storage`, or create + // a unique copy for each `Record`. + Entry meta = 5; } /* -A Tablet stores the records of a component which type is `component` and indidates as `tag`. -The records will be saved in a file which name contains `tag`. During the running period, -`num_records` will be accumulated, and `num_samples` indicates the size of sample set the +A Tablet stores the records of a component which type is `component` and +indidates as `tag`. +The records will be saved in a file which name contains `tag`. During the +running period, +`num_records` will be accumulated, and `num_samples` indicates the size of +sample set the reservoir sampling algorithm will collect. */ message Tablet { - // the kinds of the components that supported - enum Type { - kScalar = 0; - kHistogram = 1; - kGraph = 2; - } - // type of the component, different component should have different storage format. - Type component = 1; - // records the total count of records, each Write operation should increate this value. - int64 num_records = 2; - // indicate the number of instances to sample, this should be a constant value. - int32 num_samples = 3; - repeated Record records = 4; - // store a meta infomation if all the records share. - Entry meta = 5; - // the unique identification for this `Tablet`. - string tag = 6; - // one tablet might have multiple captions, for example, a scalar component might have - // two plots labeled "train" and "test". - repeated string captions = 7; + // the kinds of the components that supported + enum Type { + kScalar = 0; + kHistogram = 1; + kGraph = 2; + } + // type of the component, different component should have different storage + // format. + Type component = 1; + // records the total count of records, each Write operation should increate + // this value. + int64 total_records = 2; + // indicate the number of instances to sample, this should be a constant + // value. + int32 num_samples = 3; + repeated Record records = 4; + // store a meta infomation if all the records share. + Entry meta = 5; + // the unique identification for this `Tablet`. + string tag = 6; + // one tablet might have multiple captions, for example, a scalar component + // might have + // two plots labeled "train" and "test". + repeated string captions = 7; } /* The Storage stores all the records. */ message Storage { - // tags to Tablet, should be thread safe if fix the keys after initialization. - map tablets = 1; - string dir = 2; - int64 timestamp = 3; + // tags to Tablet, should be thread safe if fix the keys after initialization. + map tablets = 1; + string dir = 2; + int64 timestamp = 3; } \ No newline at end of file -- GitLab