storage.h 3.3 KB
Newer Older
S
superjom 已提交
1 2 3
#ifndef VISUALDL_STORAGE_H
#define VISUALDL_STORAGE_H

S
superjom 已提交
4
#include <time.h>
5
#include <map>
S
Superjom 已提交
6
#include <memory>
7
#include <string>
S
superjom 已提交
8

S
superjom 已提交
9
#include "visualdl/storage/storage.pb.h"
S
Superjom 已提交
10
#include "visualdl/utils/concurrency.h"
S
superjom 已提交
11 12 13

namespace visualdl {

14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Generate a tablet path in disk from its tag.
 */
inline std::string GenPathFromTag(const std::string &dir,
                                  const std::string &tag);

/*
 * Storage Interface. The might be a bunch of implementations, for example, a
 * MemStorage that keep a copy of all the taplets in memory, can be changed with
 * a higher performance; a DiskStorage that keep all the data in disk, apply to
 * the scenerios where memory consumption should be considered.
 */
class StorageBase {
public:
  const static std::string meta_file_name;

S
superjom 已提交
30
  enum Type { kMemory = 0, kDisk = 1 };
S
superjom 已提交
31
  // mode of the sevice, either reading or writing.
S
superjom 已提交
32
  enum Mode { kRead = 0, kWrite = 1, kNone = 2 };
S
superjom 已提交
33

34 35 36 37 38 39 40
  void SetStorage(const std::string &dir) {
    time_t t;
    time(&t);
    storage_.set_timestamp(t);
    storage_.set_dir(dir);
  }

S
Superjom 已提交
41 42
  std::string meta_path(const std::string &dir) const;
  std::string tablet_path(const std::string &dir, const std::string &tag) const;
43

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
  /*
   * Create a new Tablet storage.
   */
  virtual storage::Tablet *NewTablet(const std::string &tag,
                                     int num_samples) = 0;

  /*
   * Get a tablet from memory, this can be viewed as a cache, if the storage is
   * in disk, a hash map in memory will first load the corresponding Tablet
   * Protobuf from disk and hold all the changes.
   */
  virtual storage::Tablet *tablet(const std::string &tag) = 0;

  /*
   * Persist the data from cache to disk. Both the memory storage or disk
   * storage should write changes to disk for persistence.
   */
S
Superjom 已提交
61
  virtual void PersistToDisk(const std::string &dir) = 0;
62 63 64 65 66 67

  /*
   * Load data from disk.
   */
  virtual void LoadFromDisk(const std::string &dir) = 0;

S
superjom 已提交
68 69 70
  storage::Storage *mutable_data() { return &storage_; }
  const storage::Storage &data() { return storage_; }

71 72 73 74 75 76 77 78 79
protected:
  storage::Storage storage_;
};

/*
 * Storage in Memory, that will support quick edits on data.
 */
class MemoryStorage final : public StorageBase {
public:
S
Superjom 已提交
80 81 82 83 84
  MemoryStorage() {}
  MemoryStorage(cc::PeriodExector *executor) : executor_(executor) {}
  ~MemoryStorage() {
    if (executor_ != nullptr) executor_->Quit();
  }
85 86 87 88
  storage::Tablet *NewTablet(const std::string &tag, int num_samples) override;

  storage::Tablet *tablet(const std::string &tag) override;

S
Superjom 已提交
89
  void PersistToDisk(const std::string &dir) override;
90 91 92

  void LoadFromDisk(const std::string &dir) override;

93 94 95
  /*
   * Create a thread which will keep reading the latest data from the disk to
   * memory.
S
Superjom 已提交
96 97
   *
   * msecs: how many millisecond to sync memory and disk.
98
   */
S
Superjom 已提交
99
  void StartReadService(const std::string &dir, int msecs, std::mutex *handler);
100 101 102 103

  /*
   * Create a thread which will keep writing the latest changes from memory to
   * disk.
S
Superjom 已提交
104 105
   *
   * msecs: how many millisecond to sync memory and disk.
106
   */
S
Superjom 已提交
107
  void StartWriteSerice(const std::string &dir, int msecs, std::mutex *handler);
108

109 110
private:
  std::map<std::string, storage::Tablet> tablets_;
S
Superjom 已提交
111 112
  // TODO(ChunweiYan) remove executor here.
  cc::PeriodExector *executor_{nullptr};
113 114 115
};

}  // namespace visualdl
S
superjom 已提交
116

117
#endif  // VISUALDL_STORAGE_H