diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 437c38a8a4e2af91d350ad2a91ead49fb52f3fcf..aff239712bf1e19ae9d4933d956790e7ed2bec2c 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -101,23 +101,13 @@ int tdAppendColVal(SDataRow row, void *value, STColumn *pCol); void tdDataRowReset(SDataRow row, STSchema *pSchema); SDataRow tdDataRowDup(SDataRow row); -/* Data column definition - * +---------+---------+-----------------------+ - * | int32_t | int32_t | | - * +---------+---------+-----------------------+ - * | len | npoints | data | - * +---------+---------+-----------------------+ - */ -typedef char *SDataCol; - -/* Data columns definition - * +---------+---------+-----------------------+--------+-----------------------+ - * | int32_t | int32_t | | | | - * +---------+---------+-----------------------+--------+-----------------------+ - * | len | npoints | SDataCol | .... | SDataCol | - * +---------+---------+-----------------------+--------+-----------------------+ - */ -typedef char *SDataCols; +// ----------------- Data column structure +typedef struct SDataCol { + int64_t len; + char data[]; +} SDataCol; + +void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 419e37639257b2172ffe4eaa0c68436bb8ff6f6c..9c356b0cbc71671ee8a7d917bf18c0b988f0cb1f 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -294,6 +294,16 @@ SDataRow tdDataRowDup(SDataRow row) { return trow; } +void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter) { + int row = *iter; + + for (int i = 0; i < schemaNCols(pSchema); i++) { + // TODO + } + + *iter = row + 1; +} + /** * Return the first part length of a data row for a schema */ diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index ab10fd8e4977706de403f967c92dd9f54eff9950..7e83b843754fbc47088f536384817e653b751da4 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -26,8 +26,7 @@ extern "C" { typedef enum { TSDB_FILE_TYPE_HEAD, // .head file type TSDB_FILE_TYPE_DATA, // .data file type - TSDB_FILE_TYPE_LAST, // .last file type - TSDB_FILE_TYPE_META // .meta file type + TSDB_FILE_TYPE_LAST // .last file type } TSDB_FILE_TYPE; extern const char *tsdbFileSuffix[]; @@ -38,7 +37,6 @@ typedef struct { } SFileInfo; typedef struct { - int fd; int64_t size; // total size of the file int64_t tombSize; // unused file size } SFile; @@ -59,7 +57,7 @@ typedef struct { SFileGroup fGroup[]; } STsdbFileH; -#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) <= TSDB_FILE_TYPE_META) +#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) <= TSDB_FILE_TYPE_LAST) STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, int32_t maxRowsPerFBlock); diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 301f2978030b9fb69bca4cd5eb1f5356686fa6eb..ad9045567b69b01aa9faf47f61fb7f693239d4fd 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -12,77 +12,144 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include +#include +#include #include #include -#include #include -#include +#include +#include +#include -#include "tsdbFile.h" #include "tglobalcfg.h" +#include "tsdbFile.h" -// int64_t tsMsPerDay[] = { -// 86400000L, // TSDB_PRECISION_MILLI -// 86400000000L, // TSDB_PRECISION_MICRO -// 86400000000000L // TSDB_PRECISION_NANO -// }; +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F #define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) #define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) typedef struct { + int32_t len; + int32_t padding; // For padding purpose int64_t offset; -} SCompHeader; - -typedef struct { - int64_t uid; - int64_t last : 1; - int64_t numOfBlocks : 63; - int32_t delimiter; -} SCompInfo; - -typedef struct { - TSKEY keyFirst; - TSKEY keyLast; - int32_t numOfBlocks; - int32_t offset; } SCompIdx; +/** + * if numOfSubBlocks == -1, then the SCompBlock is a sub-block + * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to + * the data block offset and length + * if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the + * binary + */ typedef struct { + int64_t last : 1; // If the block in data file or last file + int64_t offset : 63; // Offset of data block or sub-block index depending on numOfSubBlocks + int32_t algorithm : 8; // Compression algorithm + int32_t numOfPoints : 24; // Number of total points + int32_t sversion; // Schema version + int32_t len; // Data block length or nothing + int16_t numOfSubBlocks; // Number of sub-blocks; + int16_t numOfCols; TSKEY keyFirst; TSKEY keyLast; - int64_t offset; - int32_t len; - int32_t sversion; } SCompBlock; typedef struct { - int64_t uid; -} SBlock; + int32_t delimiter; // For recovery usage + int32_t checksum; // TODO: decide if checksum logic in this file or make it one API + int64_t uid; + int32_t padding; // For padding purpose + int32_t numOfBlocks; // TODO: make the struct padding + SCompBlock blocks[]; +} SCompInfo; +// TODO: take pre-calculation into account typedef struct { - int16_t colId; - int16_t bytes; - int32_t nNullPoints; - int32_t type:8; - int32_t offset:24; - int32_t len; - // fields for pre-aggregate - // TODO: pre-aggregation should be seperated - int64_t sum; - int64_t max; - int64_t min; - int16_t maxIdx; - int16_t minIdx; -} SField; + int16_t colId; // Column ID + int16_t len; // Column length + int32_t type : 8; + int32_t offset : 24; +} SCompCol; + +// TODO: Take recover into account +typedef struct { + int32_t delimiter; // For recovery usage + int32_t numOfCols; // For recovery usage + int64_t uid; // For recovery usage + SCompCol cols[]; +} SCompData; const char *tsdbFileSuffix[] = { ".head", // TSDB_FILE_TYPE_HEAD ".data", // TSDB_FILE_TYPE_DATA - ".last", // TSDB_FILE_TYPE_LAST - ".meta" // TSDB_FILE_TYPE_META + ".last" // TSDB_FILE_TYPE_LAST }; +static int tsdbWriteFileHead(int fd) { + char head[TSDB_FILE_HEAD_SIZE] = "\0"; + + lseek(fd, 0, SEEK_SET); + if (write(fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1; + + return 0; +} + +static int tsdbWriteHeadFileIdx(int fd, int maxTables) { + int size = sizeof(SCompIdx) * maxTables; + void *buf = calloc(1, size); + if (buf == NULL) return -1; + + if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + free(buf); + return NULL; + } + + if (write(fd, buf, size) < 0) { + free(buf); + return -1; + } + + return 0; +} + +static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables) { + char fname[128] = "\0"; + sprintf(fname, "%s/f%d%s", dataDir, fileId, tsdbFileSuffix[type]); + if (access(fname, F_OK) == 0) { + // File already exists + return -1; + } + + int fd = open(fname, O_RDWR | O_CREAT, 0755); + if (fd < 0) return -1; + + if (tsdbWriteFileHead(fd) < 0) { + close(fd); + return -1; + } + + if (type == TSDB_FILE_TYPE_LAST) { + if (tsdbWriteHeadFileIdx(fd, maxTables) < 0) { + close(fd); + return -1; + } + } + + close(fd); + + return 0; +} + +// Create a file group with fileId and return a SFileGroup object +static int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup) { + // tsdbCreateFile() + + return 0; +} + /** * Initialize the TSDB file handle */ @@ -105,7 +172,7 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32 } struct dirent *dp; - char fname[256]; + char fname[256]; while ((dp = readdir(dir)) != NULL) { if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; if (true /* check if the file is the .head file */) { diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 073321816db29939f5ba28400ad5ad5f598bfa29..5104c664dcade8637e76b5aa3d65703dcbc3b672 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -612,9 +612,6 @@ static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo) { rmdir(dirName); - char *metaFname = tsdbGetFileName(pRepo->rootDir, "tsdb", TSDB_FILE_TYPE_META); - remove(metaFname); - return 0; }