diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h
index 437c38a8a4e2af91d350ad2a91ead49fb52f3fcf..aff239712bf1e19ae9d4933d956790e7ed2bec2c 100644
--- a/src/common/inc/dataformat.h
+++ b/src/common/inc/dataformat.h
@@ -101,23 +101,13 @@ int tdAppendColVal(SDataRow row, void *value, STColumn *pCol);
void tdDataRowReset(SDataRow row, STSchema *pSchema);
SDataRow tdDataRowDup(SDataRow row);
-/* Data column definition
- * +---------+---------+-----------------------+
- * | int32_t | int32_t | |
- * +---------+---------+-----------------------+
- * | len | npoints | data |
- * +---------+---------+-----------------------+
- */
-typedef char *SDataCol;
-
-/* Data columns definition
- * +---------+---------+-----------------------+--------+-----------------------+
- * | int32_t | int32_t | | | |
- * +---------+---------+-----------------------+--------+-----------------------+
- * | len | npoints | SDataCol | .... | SDataCol |
- * +---------+---------+-----------------------+--------+-----------------------+
- */
-typedef char *SDataCols;
+// ----------------- Data column structure
+typedef struct SDataCol {
+ int64_t len;
+ char data[];
+} SDataCol;
+
+void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter);
#ifdef __cplusplus
}
diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c
index 419e37639257b2172ffe4eaa0c68436bb8ff6f6c..9c356b0cbc71671ee8a7d917bf18c0b988f0cb1f 100644
--- a/src/common/src/dataformat.c
+++ b/src/common/src/dataformat.c
@@ -294,6 +294,16 @@ SDataRow tdDataRowDup(SDataRow row) {
return trow;
}
+void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter) {
+ int row = *iter;
+
+ for (int i = 0; i < schemaNCols(pSchema); i++) {
+ // TODO
+ }
+
+ *iter = row + 1;
+}
+
/**
* Return the first part length of a data row for a schema
*/
diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h
index ab10fd8e4977706de403f967c92dd9f54eff9950..7e83b843754fbc47088f536384817e653b751da4 100644
--- a/src/vnode/tsdb/inc/tsdbFile.h
+++ b/src/vnode/tsdb/inc/tsdbFile.h
@@ -26,8 +26,7 @@ extern "C" {
typedef enum {
TSDB_FILE_TYPE_HEAD, // .head file type
TSDB_FILE_TYPE_DATA, // .data file type
- TSDB_FILE_TYPE_LAST, // .last file type
- TSDB_FILE_TYPE_META // .meta file type
+ TSDB_FILE_TYPE_LAST // .last file type
} TSDB_FILE_TYPE;
extern const char *tsdbFileSuffix[];
@@ -38,7 +37,6 @@ typedef struct {
} SFileInfo;
typedef struct {
- int fd;
int64_t size; // total size of the file
int64_t tombSize; // unused file size
} SFile;
@@ -59,7 +57,7 @@ typedef struct {
SFileGroup fGroup[];
} STsdbFileH;
-#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) <= TSDB_FILE_TYPE_META)
+#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) <= TSDB_FILE_TYPE_LAST)
STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock,
int32_t maxRowsPerFBlock);
diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c
index 301f2978030b9fb69bca4cd5eb1f5356686fa6eb..ad9045567b69b01aa9faf47f61fb7f693239d4fd 100644
--- a/src/vnode/tsdb/src/tsdbFile.c
+++ b/src/vnode/tsdb/src/tsdbFile.c
@@ -12,77 +12,144 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
+#include
+#include
+#include
#include
#include
-#include
#include
-#include
+#include
+#include
+#include
-#include "tsdbFile.h"
#include "tglobalcfg.h"
+#include "tsdbFile.h"
-// int64_t tsMsPerDay[] = {
-// 86400000L, // TSDB_PRECISION_MILLI
-// 86400000000L, // TSDB_PRECISION_MICRO
-// 86400000000000L // TSDB_PRECISION_NANO
-// };
+#define TSDB_FILE_HEAD_SIZE 512
+#define TSDB_FILE_DELIMITER 0xF00AFA0F
#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
typedef struct {
+ int32_t len;
+ int32_t padding; // For padding purpose
int64_t offset;
-} SCompHeader;
-
-typedef struct {
- int64_t uid;
- int64_t last : 1;
- int64_t numOfBlocks : 63;
- int32_t delimiter;
-} SCompInfo;
-
-typedef struct {
- TSKEY keyFirst;
- TSKEY keyLast;
- int32_t numOfBlocks;
- int32_t offset;
} SCompIdx;
+/**
+ * if numOfSubBlocks == -1, then the SCompBlock is a sub-block
+ * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to
+ * the data block offset and length
+ * if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the
+ * binary
+ */
typedef struct {
+ int64_t last : 1; // If the block in data file or last file
+ int64_t offset : 63; // Offset of data block or sub-block index depending on numOfSubBlocks
+ int32_t algorithm : 8; // Compression algorithm
+ int32_t numOfPoints : 24; // Number of total points
+ int32_t sversion; // Schema version
+ int32_t len; // Data block length or nothing
+ int16_t numOfSubBlocks; // Number of sub-blocks;
+ int16_t numOfCols;
TSKEY keyFirst;
TSKEY keyLast;
- int64_t offset;
- int32_t len;
- int32_t sversion;
} SCompBlock;
typedef struct {
- int64_t uid;
-} SBlock;
+ int32_t delimiter; // For recovery usage
+ int32_t checksum; // TODO: decide if checksum logic in this file or make it one API
+ int64_t uid;
+ int32_t padding; // For padding purpose
+ int32_t numOfBlocks; // TODO: make the struct padding
+ SCompBlock blocks[];
+} SCompInfo;
+// TODO: take pre-calculation into account
typedef struct {
- int16_t colId;
- int16_t bytes;
- int32_t nNullPoints;
- int32_t type:8;
- int32_t offset:24;
- int32_t len;
- // fields for pre-aggregate
- // TODO: pre-aggregation should be seperated
- int64_t sum;
- int64_t max;
- int64_t min;
- int16_t maxIdx;
- int16_t minIdx;
-} SField;
+ int16_t colId; // Column ID
+ int16_t len; // Column length
+ int32_t type : 8;
+ int32_t offset : 24;
+} SCompCol;
+
+// TODO: Take recover into account
+typedef struct {
+ int32_t delimiter; // For recovery usage
+ int32_t numOfCols; // For recovery usage
+ int64_t uid; // For recovery usage
+ SCompCol cols[];
+} SCompData;
const char *tsdbFileSuffix[] = {
".head", // TSDB_FILE_TYPE_HEAD
".data", // TSDB_FILE_TYPE_DATA
- ".last", // TSDB_FILE_TYPE_LAST
- ".meta" // TSDB_FILE_TYPE_META
+ ".last" // TSDB_FILE_TYPE_LAST
};
+static int tsdbWriteFileHead(int fd) {
+ char head[TSDB_FILE_HEAD_SIZE] = "\0";
+
+ lseek(fd, 0, SEEK_SET);
+ if (write(fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1;
+
+ return 0;
+}
+
+static int tsdbWriteHeadFileIdx(int fd, int maxTables) {
+ int size = sizeof(SCompIdx) * maxTables;
+ void *buf = calloc(1, size);
+ if (buf == NULL) return -1;
+
+ if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {
+ free(buf);
+ return NULL;
+ }
+
+ if (write(fd, buf, size) < 0) {
+ free(buf);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables) {
+ char fname[128] = "\0";
+ sprintf(fname, "%s/f%d%s", dataDir, fileId, tsdbFileSuffix[type]);
+ if (access(fname, F_OK) == 0) {
+ // File already exists
+ return -1;
+ }
+
+ int fd = open(fname, O_RDWR | O_CREAT, 0755);
+ if (fd < 0) return -1;
+
+ if (tsdbWriteFileHead(fd) < 0) {
+ close(fd);
+ return -1;
+ }
+
+ if (type == TSDB_FILE_TYPE_LAST) {
+ if (tsdbWriteHeadFileIdx(fd, maxTables) < 0) {
+ close(fd);
+ return -1;
+ }
+ }
+
+ close(fd);
+
+ return 0;
+}
+
+// Create a file group with fileId and return a SFileGroup object
+static int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup) {
+ // tsdbCreateFile()
+
+ return 0;
+}
+
/**
* Initialize the TSDB file handle
*/
@@ -105,7 +172,7 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32
}
struct dirent *dp;
- char fname[256];
+ char fname[256];
while ((dp = readdir(dir)) != NULL) {
if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue;
if (true /* check if the file is the .head file */) {
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index 073321816db29939f5ba28400ad5ad5f598bfa29..5104c664dcade8637e76b5aa3d65703dcbc3b672 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -612,9 +612,6 @@ static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo) {
rmdir(dirName);
- char *metaFname = tsdbGetFileName(pRepo->rootDir, "tsdb", TSDB_FILE_TYPE_META);
- remove(metaFname);
-
return 0;
}