storage.proto

syntax = "proto3";
package storage;

enum DataType {
    // single entry
    kInt32 = 0;
    kInt64 = 1;
    kFloat = 2;
    kDouble = 3;
    kString = 4;
    kBool = 5;
    // entrys
    kInt64s = 6;
    kFloats = 7;
    kDoubles = 8;
    kStrings = 9;
    kInt32s = 10;
    kBools = 11;

    kUnknown = 12;
}

// A data array, which type is `type`.
message Entry {
    // if all the entries in a record share the same data type, ignore this field
    // and store type to `dtype` in `Record`.
    DataType dtype = 1;
    // single element
    int32 i32 = 2;
    int64 i64 = 3;
    string s = 4;
    float f = 5;
    double d = 6;
    bool b = 7;
    // array
    repeated int64 i64s = 8;
    repeated float fs = 9;
    repeated double ds = 10;
    repeated int32 i32s = 11;
    repeated bool bs = 12;
}

message Record {
    Entry data = 1;
    int64 timestamp = 2;
    // store the count of writing operations to the tablet.
    int64 id = 3;
    DataType dtype = 4;
    // shape or some other meta infomation for this record, if all the records
    // share the same meta, just store one copy of meta in `Storage`, or create
    // a unique copy for each `Record`.
    Entry meta = 5;
}

/*
A Tablet stores the records of a component which type is `component` and indidates as `tag`.
The records will be saved in a file which name contains `tag`. During the running period,
`num_records` will be accumulated, and `num_samples` indicates the size of sample set the
reservoir sampling algorithm will collect.
*/
message Tablet {
    // the kinds of the components that supported
    enum Type {
        kScalar = 0;
        kHistogram = 1;
        kGraph = 2;
    }
    // type of the component, different component should have different storage format.
    Type component = 1;
    // records the total count of records, each Write operation should increate this value.
    int64 num_records = 2;
    // indicate the number of instances to sample, this should be a constant value.
    int32 num_samples = 3;
    repeated Record records = 4;
    // store a meta infomation if all the records share.
    Entry meta = 5;
    // the unique identification for this `Tablet`.
    string tag = 6;
}

/*
The Storage stores all the records.
*/
message Storage {
    // tags to Tablet, should be thread safe if fix the keys after initialization.
    map<string, Tablet> tablets = 1;
    string dir = 2;
    int64 timestamp = 3;
}