提交 9fa67eff 编写于 作者: Y Yan Chunwei 提交者: GitHub

Merge pull request #22 from Superjom/feature/logic_layer_init

......@@ -5,9 +5,10 @@ set(CMAKE_CXX_STANDARD 11)
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
include_directories(thirdparty/local/include)
link_directories(thirdparty/local/lib)
add_subdirectory(thirdparty/pybind11-2.2.1)
link_directories(thirdparty/local/lib)
set(SOURCE_FILES
visualdl/backend/storage/storage.cc visualdl/backend/storage/storage.h
visualdl/backend/storage/storage.pb.h
......@@ -16,4 +17,13 @@ set(SOURCE_FILES
add_library(storage visualdl/backend/storage/storage.cc
visualdl/backend/storage/storage.pb.cc)
add_library(c_api visualdl/backend/logic/c_api.cc)
add_library(sdk visualdl/backend/logic/sdk.cc)
add_library(im visualdl/backend/logic/im.cc)
add_executable(vl_test
visualdl/backend/test.cc
visualdl/backend/logic/im_test.cc)
target_link_libraries(vl_test storage im gtest glog protobuf gflags)
add_executable(VisualDL ${SOURCE_FILES})
#include <ctime>
#include <glog/logging.h>
#include "visualdl/backend/logic/im.h"
namespace visualdl {
/*
* @num_samples: number of instances to sample
* @size: counter of the records.
* @returns: id of the instance to replace, if drop this instance, return -1.
*/
int ReserviorSample(int num_samples, int num_records) {
if (num_records <= num_samples) {
return num_records;
}
std::srand(std::time(0));
float prob = static_cast<float>(std::rand()) / RAND_MAX;
float receive_prob = static_cast<float>(num_samples) / num_records;
if (prob < receive_prob) {
int offset2replace = std::rand() % num_samples;
return offset2replace;
}
return -1;
}
void InformationMaintainer::SetPersistDest(const std::string &path) {
CHECK(storage_.mutable_data()->dir().empty())
<< "duplicate set storage's path";
storage_.mutable_data()->set_dir(path);
}
storage::Tablet *InformationMaintainer::AddTablet(const std::string &tag,
int num_samples) {
auto *tablet = storage_.Find(tag);
if (!tablet) {
tablet = storage_.Add(tag, num_samples);
}
return tablet;
}
void InformationMaintainer::AddRecord(const std::string &tag,
storage::Tablet::Type type,
const std::string &data,
storage::DataType dtype) {
auto *tablet = storage_.Find(tag);
CHECK(tablet);
auto num_records = tablet->num_records();
const auto num_samples = tablet->num_samples();
const auto offset = ReserviorSample(num_samples, num_records + 1);
if (offset < 0)
return;
storage::Record *record;
if (offset >= num_records) {
record = storage_.NewRecord(tag);
} else {
record = storage_.GetRecord(tag, offset);
}
record->set_dtype(dtype);
record->ParseFromString(data);
tablet->set_num_records(num_records + 1);
}
void InformationMaintainer::PersistToDisk() {
CHECK(!storage_.data().dir().empty()) << "path of storage should be set";
storage_.Save(storage_.data().dir());
}
} // namespace visualdl
#ifndef VISUALDL_BACKEND_LOGIC_IM_H
#define VISUALDL_BACKEND_LOGIC_IM_H
#include <string>
#include "visualdl/backend/storage/storage.h"
namespace visualdl {
/*
* Maintain the Storage singleton in memory, pre-compute some the statical
* information to help visualizaton.
*/
class InformationMaintainer final {
public:
InformationMaintainer() {}
static InformationMaintainer &Global() {
static InformationMaintainer *x = new InformationMaintainer();
return *x;
}
/*
* Set the disk path to store the Storage object.
*/
void SetPersistDest(const std::string &path);
storage::Tablet *AddTablet(const std::string &tag, int num_samples);
/*
* @tag: tag of the target Tablet.
* @type: type of target Tablet.
* @data: serialized protobuf message.
*
* NOTE pass in the serialized protobuf message will trigger copying, but
* simpler to support different Tablet data formats.
*/
void AddRecord(const std::string &tag, storage::Tablet::Type type,
const std::string &data,
storage::DataType dtype = storage::DataType::kUnknown);
/*
* Save the Storage Protobuf to disk.
*/
void PersistToDisk();
Storage &storage() { return storage_; }
private:
Storage storage_;
};
} // namespace visualdl
#endif // VISUALDL_BACKEND_LOGIC_IM_H
#include "visualdl/backend/logic/im.h"
#include "gtest/gtest.h"
namespace visualdl {
class ImTester : public ::testing::Test {
protected:
void SetUp() override {}
InformationMaintainer &im = InformationMaintainer::Global();
};
TEST_F(ImTester, AddTablet) { im.AddTablet("tag0", 20); }
TEST_F(ImTester, AddRecord) {
storage::Record rcd;
rcd.set_dtype(storage::DataType::kInt32s);
for (int i = 0; i < 100; i++) {
for (int j = 0; j < 10; j++) {
rcd.mutable_data()->add_i32s(i * 20 + j);
}
im.AddRecord("tag0", storage::Tablet::Type::Tablet_Type_kGraph,
rcd.SerializeAsString());
}
ASSERT_EQ(im.storage().Find("tag0")->records_size(), 20UL);
}
} // namespace visualdl
......@@ -5,18 +5,38 @@
namespace visualdl {
storage::Tablet *Storage::Add(const std::string &tag) {
return &proto_.mutable_tablets()->at(tag);
storage::Tablet *Storage::Add(const std::string &tag, int num_samples) {
auto *tablet = &(*proto_.mutable_tablets())[tag];
tablet->set_num_samples(num_samples);
return tablet;
}
const storage::Tablet *Storage::Find(const std::string &tag) const {
auto it = proto_.tablets().find(tag);
storage::Tablet *Storage::Find(const std::string &tag) {
auto it = proto_.mutable_tablets()->find(tag);
if (it != proto_.tablets().end()) {
return &it->second;
}
return nullptr;
}
storage::Record *Storage::NewRecord(const std::string &tag) {
auto *tablet = Find(tag);
CHECK(tablet) << "Tablet" << tag << " should be create first";
auto *record = tablet->mutable_records()->Add();
// increase num_records
int num_records = tablet->num_records();
tablet->set_num_records(num_records + 1);
return record;
}
storage::Record *Storage::GetRecord(const std::string &tag, int offset) {
auto *tablet = Find(tag);
CHECK(tablet) << "Tablet" << tag << " should be create first";
auto num_records = tablet->num_records();
CHECK_LT(offset, num_records) << "invalid offset";
return tablet->mutable_records()->Mutable(offset);
}
void Storage::Save(const std::string &path) const {
std::ofstream file(path, file.binary | file.out);
CHECK(file.is_open()) << "can't open path " << path;
......@@ -34,9 +54,7 @@ void Storage::Load(const std::string &path) {
DeSerialize(buffer);
}
std::string Storage::Serialize() const {
return proto_.SerializeAsString();
}
std::string Storage::Serialize() const { return proto_.SerializeAsString(); }
void Storage::DeSerialize(const std::string &data) {
proto_.ParseFromString(data);
......
......@@ -9,24 +9,33 @@
namespace visualdl {
class Storage final {
public:
/*
* There should be only one Storage instance in memory.
*/
Storage &Global() {
static Storage *instance = new Storage();
return *instance;
public:
Storage() {
// set time stamp
time_t time0;
time(&time0);
proto_.set_timestamp(time0);
}
/*
* Add a new tablet named `tag`, the newly added instance will be returned.
*/
storage::Tablet *Add(const std::string &tag);
storage::Tablet *Add(const std::string &tag, int num_samples);
/*
* Search the tablet named `tag`, if not exist, return nullptr.
*/
const storage::Tablet *Find(const std::string &tag) const;
storage::Tablet *Find(const std::string &tag);
/*
* Append a new record to the tail of tablet.
*/
storage::Record *NewRecord(const std::string &tag);
/*
* Get a record at `offset`, if the offset is not valid, yield a failed CHECK.
*/
storage::Record *GetRecord(const std::string &tag, int offset);
/*
* Serialize this object to string and save it to a file.
......@@ -38,7 +47,11 @@ class Storage final {
*/
void Load(const std::string &path);
protected:
storage::Storage *mutable_data() { return &proto_; }
const storage::Storage &data() { return proto_; }
protected:
/*
* Serialize the Storage instance to string.
*/
......@@ -49,17 +62,10 @@ class Storage final {
*/
void DeSerialize(const std::string &data);
Storage() {
// set time stamp
time_t time0;
time(&time0);
proto_.set_timestamp(time0);
}
private:
private:
storage::Storage proto_;
};
} // namespace visualdl
#endif //VISUALDL_STORAGE_H
#endif // VISUALDL_STORAGE_H
......@@ -16,6 +16,8 @@ enum DataType {
kStrings = 9;
kInt32s = 10;
kBools = 11;
kUnknown = 12;
}
// A data array, which type is `type`.
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册