diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 9b8c86658d4de65c2e620ff8a249044f3fcaf6cd..85b1d3d206f07647c2fd68c9e61e7f8d7367f290 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -253,6 +253,7 @@ static __compar_fn_t getKeyComparator(int32_t keyType) { comparFn = compareInt32Val; break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; case TSDB_DATA_TYPE_BOOL: diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index 267b462b91dbd07ac512b02681fb3efc6fdeffab..5cd7956b1e247de1e6207fc7e4a3dd125c5d1bd8 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -75,6 +75,7 @@ typedef struct { typedef struct { TSDB_TABLE_TYPE type; STableId tableId; + int32_t sversion; int64_t superUid; STSchema * schema; STSchema * tagSchema; diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 7b3e19d0b981d8f1c4d78297b087c55e61984309..ab10fd8e4977706de403f967c92dd9f54eff9950 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -23,8 +23,6 @@ extern "C" { #endif -typedef int32_t file_id_t; - typedef enum { TSDB_FILE_TYPE_HEAD, // .head file type TSDB_FILE_TYPE_DATA, // .data file type @@ -40,19 +38,33 @@ typedef struct { } SFileInfo; typedef struct { - char * fname; - SFileInfo fInfo; -} SFILE; + int fd; + int64_t size; // total size of the file + int64_t tombSize; // unused file size +} SFile; -// typedef struct { -// int64_t offset; -// int64_t skey; -// int64_t ekey; -// int16_t numOfBlocks; -// } SDataBlock; +typedef struct { + int32_t fileId; + SFile fhead; + SFile fdata; + SFile flast; +} SFileGroup; + +// TSDB file handle +typedef struct { + int32_t daysPerFile; + int32_t keep; + int32_t minRowPerFBlock; + int32_t maxRowsPerFBlock; + SFileGroup fGroup[]; +} STsdbFileH; #define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) <= TSDB_FILE_TYPE_META) +STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, + int32_t maxRowsPerFBlock); +void tsdbCloseFile(STsdbFileH *pFileH); + char *tsdbGetFileName(char *dirName, char *fname, TSDB_FILE_TYPE type); #ifdef __cplusplus diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h index be7c7d0406770ba143d847be5f3bb4c411d3260c..99caa1a442984dd74e8c94be056690771976985f 100644 --- a/src/vnode/tsdb/inc/tsdbMeta.h +++ b/src/vnode/tsdb/inc/tsdbMeta.h @@ -38,6 +38,7 @@ typedef struct STable { TSDB_TABLE_TYPE type; STableId tableId; int32_t superUid; // Super table UID + int32_t sversion; STSchema * schema; STSchema * tagSchema; SDataRow tagVal; diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index b977a51b51481294871413390641c90559b9cdff..301f2978030b9fb69bca4cd5eb1f5356686fa6eb 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -14,9 +14,21 @@ */ #include #include +#include #include +#include #include "tsdbFile.h" +#include "tglobalcfg.h" + +// int64_t tsMsPerDay[] = { +// 86400000L, // TSDB_PRECISION_MILLI +// 86400000000L, // TSDB_PRECISION_MICRO +// 86400000000000L // TSDB_PRECISION_NANO +// }; + +#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) +#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) typedef struct { int64_t offset; @@ -71,6 +83,55 @@ const char *tsdbFileSuffix[] = { ".meta" // TSDB_FILE_TYPE_META }; +/** + * Initialize the TSDB file handle + */ +STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, + int32_t maxRowsPerFBlock) { + STsdbFileH *pTsdbFileH = + (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * tsdbGetMaxNumOfFiles(keep, daysPerFile)); + if (pTsdbFileH == NULL) return NULL; + + pTsdbFileH->daysPerFile = daysPerFile; + pTsdbFileH->keep = keep; + pTsdbFileH->minRowPerFBlock = minRowsPerFBlock; + pTsdbFileH->maxRowsPerFBlock = maxRowsPerFBlock; + + // Open the directory to read information of each file + DIR *dir = opendir(dataDir); + if (dir == NULL) { + free(pTsdbFileH); + return NULL; + } + + struct dirent *dp; + char fname[256]; + while ((dp = readdir(dir)) != NULL) { + if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; + if (true /* check if the file is the .head file */) { + int fileId = 0; + int vgId = 0; + sscanf(dp->d_name, "v%df%d.head", &vgId, &fileId); + // TODO + + // Open head file + + // Open data file + + // Open last file + } + } + + return pTsdbFileH; +} + +/** + * Closet the file handle + */ +void tsdbCloseFile(STsdbFileH *pFileH) { + // TODO +} + char *tsdbGetFileName(char *dirName, char *fname, TSDB_FILE_TYPE type) { if (!IS_VALID_TSDB_FILE_TYPE(type)) return NULL; @@ -79,4 +140,10 @@ char *tsdbGetFileName(char *dirName, char *fname, TSDB_FILE_TYPE type) { sprintf(fileName, "%s/%s%s", dirName, fname, tsdbFileSuffix[type]); return fileName; +} + +static void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, + TSKEY *maxKey) { + *minKey = fileId * daysPerFile * tsMsPerDay[precision]; + *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 0f70875c635feb05f2bac8e0b7261cc74cf04b07..f26f0e06f2df02d75cae406cd120f6e55e12190e 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -308,7 +308,6 @@ int tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) { } int tsdbDropTable(tsdb_repo_t *repo, STableId tableId) { - // TODO if (repo == NULL) return -1; STsdbRepo *pRepo = (STsdbRepo *)repo; diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 5c5c5c50f0b83b7ec41c81443a9323f0bbd8727f..d460530ea5524f6913b5d27938a6506d18161fe9 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -18,6 +18,7 @@ static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable); static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbEstimateTableEncodeSize(STable *pTable); +static char * getTupleKey(const void *data); /** * Encode a TSDB table object as a binary content @@ -153,7 +154,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { super->tagSchema = tdDupSchema(pCfg->tagSchema); super->tagVal = tdDataRowDup(pCfg->tagValues); super->content.pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, - 0, NULL); // Allow duplicate key, no lock + 0, getTupleKey); // Allow duplicate key, no lock if (super->content.pIndex == NULL) { tdFreeSchema(super->schema); @@ -183,7 +184,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { table->superUid = -1; table->schema = tdDupSchema(pCfg->schema); } - table->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, 0, 8, 0, 0, NULL); + table->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); if (newSuper) tsdbAddTableToMeta(pMeta, super); tsdbAddTableToMeta(pMeta, table); @@ -319,4 +320,10 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { static int tsdbEstimateTableEncodeSize(STable *pTable) { // TODO return 0; +} + +static char *getTupleKey(const void * data) { + SDataRow row = (SDataRow)data; + + return dataRowAt(row, TD_DATA_ROW_HEAD_SIZE); } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMetaFile.c b/src/vnode/tsdb/src/tsdbMetaFile.c index ee173d7d7169b19d007eb5cc8eeb37644ec13ed7..70ae0611068dfd9e743524153c9a878b6bcad432 100644 --- a/src/vnode/tsdb/src/tsdbMetaFile.c +++ b/src/vnode/tsdb/src/tsdbMetaFile.c @@ -83,7 +83,7 @@ int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t co } // TODO: make below a function to implement - if (fseek(mfh->fd, info.offset, SEEK_CUR) < 0) { + if (lseek(mfh->fd, info.offset, SEEK_CUR) < 0) { return -1; } @@ -114,7 +114,7 @@ int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) { // Remove record from file info.offset = -info.offset; - if (fseek(mfh->fd, -info.offset, SEEK_CUR) < 0) { + if (lseek(mfh->fd, -info.offset, SEEK_CUR) < 0) { return -1; } @@ -149,7 +149,7 @@ int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t co mfh->size += contLen; } - if (fseek(mfh->fd, -info.offset, SEEK_CUR) < 0) { + if (lseek(mfh->fd, -info.offset, SEEK_CUR) < 0) { return -1; } @@ -212,7 +212,7 @@ static int tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh) { return -1; } - if (fseek(fd, TSDB_META_FILE_HEADER_SIZE, SEEK_SET) < 0) { + if (lseek(fd, TSDB_META_FILE_HEADER_SIZE, SEEK_SET) < 0) { // TODO: deal with the error close(fd); return -1; diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 7b09fdfcdebc0828b08e22f30157a413a553b3ab..65522c12621ded5dab76277c985b1b96a994ce9f 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -48,7 +48,7 @@ TEST(TsdbTest, createRepo) { for (int j = 0; j < schemaNCols(schema); j++) { if (j == 0) { // Just for timestamp - tdAppendColVal(row, (void *)(&time), schemaColAt(schema, j)); + tdAppendColVal(row, (void *)(&ttime), schemaColAt(schema, j)); } else { // For int int val = 10; tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j)); @@ -61,5 +61,7 @@ TEST(TsdbTest, createRepo) { pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; tsdbInsertData(pRepo, pMsg); + + int k = 0; }