From 3a759852fffc46fa6d808decab7f00eea8473d3c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 09:39:05 +0000 Subject: [PATCH] TD-353 --- src/common/inc/tdataformat.h | 4 +- src/common/src/tdataformat.c | 25 +++--- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbFile.c | 21 ++--- src/tsdb/src/tsdbMeta.c | 56 +++++++------ src/tsdb/src/tsdbRWHelper.c | 24 +++--- src/util/inc/tcoding.h | 145 ++++++++++++++++++++------------- src/util/src/tkvstore.c | 34 ++++---- src/util/tests/codingTests.cpp | 96 ++++++++++++++-------- 9 files changed, 241 insertions(+), 166 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index ca9fd3ee33..b696aee2eb 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -83,7 +83,7 @@ typedef struct { #define tdFreeSchema(s) tfree((s)) STSchema *tdDupSchema(STSchema *pSchema); -void * tdEncodeSchema(void *buf, STSchema *pSchema); +int tdEncodeSchema(void **buf, STSchema *pSchema); void * tdDecodeSchema(void *buf, STSchema **pRSchema); static FORCE_INLINE int comparColId(const void *key1, const void *key2) { @@ -288,7 +288,7 @@ typedef struct { SKVRow tdKVRowDup(SKVRow row); int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); -void * tdEncodeKVRow(void *buf, SKVRow row); +int tdEncodeKVRow(void **buf, SKVRow row); void * tdDecodeKVRow(void *buf, SKVRow *row); static FORCE_INLINE int comparTagId(const void *key1, const void *key2) { diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 77af404eda..d17ce585cd 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -34,18 +34,19 @@ STSchema *tdDupSchema(STSchema *pSchema) { /** * Encode a schema to dst, and return the next pointer */ -void *tdEncodeSchema(void *buf, STSchema *pSchema) { - buf = taosEncodeFixedI32(buf, schemaVersion(pSchema)); - buf = taosEncodeFixedI32(buf, schemaNCols(pSchema)); +int tdEncodeSchema(void **buf, STSchema *pSchema) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, schemaVersion(pSchema)); + tlen += taosEncodeFixedI32(buf, schemaNCols(pSchema)); for (int i = 0; i < schemaNCols(pSchema); i++) { STColumn *pCol = schemaColAt(pSchema, i); - buf = taosEncodeFixedI8(buf, colType(pCol)); - buf = taosEncodeFixedI16(buf, colColId(pCol)); - buf = taosEncodeFixedI32(buf, colBytes(pCol)); + tlen += taosEncodeFixedI8(buf, colType(pCol)); + tlen += taosEncodeFixedI16(buf, colColId(pCol)); + tlen += taosEncodeFixedI32(buf, colBytes(pCol)); } - return buf; + return tlen; } /** @@ -605,10 +606,14 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { return 0; } -void *tdEncodeKVRow(void *buf, SKVRow row) { +int tdEncodeKVRow(void **buf, SKVRow row) { // May change the encode purpose - kvRowCpy(buf, row); - return POINTER_SHIFT(buf, kvRowLen(row)); + if (buf != NULL) { + kvRowCpy(*buf, row); + *buf = POINTER_SHIFT(*buf, kvRowLen(row)); + } + + return kvRowLen(row); } void *tdDecodeKVRow(void *buf, SKVRow *row) { diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 40f79e544f..44e8002ac1 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -330,7 +330,7 @@ int tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type); SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags); void tsdbFitRetention(STsdbRepo* pRepo); int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); -void* tsdbEncodeSFileInfo(void* buf, const STsdbFileInfo* pInfo); +int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); int tsdbCpySFile(SFile* src, SFile* dst); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 4556e79db4..e268aa1d65 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -302,8 +302,8 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { char buf[TSDB_FILE_HEAD_SIZE] = "\0"; void *pBuf = (void *)buf; - pBuf = taosEncodeFixedU32(pBuf, version); - pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); + taosEncodeFixedU32(pBuf, version); + tsdbEncodeSFileInfo(pBuf, &(pFile->info)); taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); @@ -321,15 +321,16 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { return 0; } -void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { - buf = taosEncodeFixedU32(buf, pInfo->offset); - buf = taosEncodeFixedU32(buf, pInfo->len); - buf = taosEncodeFixedU64(buf, pInfo->size); - buf = taosEncodeFixedU64(buf, pInfo->tombSize); - buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); - buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); +int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { + int tlen = 0; + tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU32(buf, pInfo->len); + tlen += taosEncodeFixedU64(buf, pInfo->size); + tlen += taosEncodeFixedU64(buf, pInfo->tombSize); + tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); + tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks); - return buf; + return tlen; } void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 4acb0a3625..9eb5d10371 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -42,9 +42,9 @@ static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); -static void * tsdbEncodeTableName(void *buf, tstr *name); +static int tsdbEncodeTableName(void **buf, tstr *name); static void * tsdbDecodeTableName(void *buf, tstr **name); -static void * tsdbEncodeTable(void *buf, STable *pTable); +static int tsdbEncodeTable(void **buf, STable *pTable); static void * tsdbDecodeTable(void *buf, STable **pRTable); // ------------------ OUTER FUNCTIONS ------------------ @@ -85,6 +85,9 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { } if (tsdbAddTableToMeta(pRepo, table, true) < 0) goto _err; + int tlen = tsdbEncodeTable(NULL, pTable); + ASSERT(tlen > 0); + // // Write to meta file // int bufLen = 0; // char *buf = malloc(1024 * 1024); @@ -472,10 +475,11 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { } if (isChanged) { - char *buf = malloc(1024 * 1024); - tsdbEncodeTable(buf, pTable); - // tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); - free(buf); + // TODO + // char *buf = malloc(1024 * 1024); + // tsdbEncodeTable(buf, pTable); + // // tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); + // free(buf); } return TSDB_CODE_SUCCESS; @@ -999,14 +1003,17 @@ void tsdbClearTableCfg(STableCfg *config) { } } -static void *tsdbEncodeTableName(void *buf, tstr *name) { - void *pBuf = buf; +static int tsdbEncodeTableName(void **buf, tstr *name) { + int tlen = 0; - pBuf = taosEncodeFixedI16(pBuf, name->len); - memcpy(pBuf, name->data, name->len); - pBuf = POINTER_SHIFT(pBuf, name->len); + tlen += taosEncodeFixedI16(buf, name->len); + if (buf != NULL) { + memcpy(*buf, name->data, name->len); + *buf = POINTER_SHIFT(*buf, name->len); + } + tlen += name->len; - return pBuf; + return tlen; } static void *tsdbDecodeTableName(void *buf, tstr **name) { @@ -1025,33 +1032,34 @@ static void *tsdbDecodeTableName(void *buf, tstr **name) { return buf; } -static void *tsdbEncodeTable(void *buf, STable *pTable) { +static int tsdbEncodeTable(void **buf, STable *pTable) { ASSERT(pTable != NULL); + int tlen = 0; - buf = taosEncodeFixedU8(buf, pTable->type); - buf = tsdbEncodeTableName(buf, pTable->name); - buf = taosEncodeFixedU64(buf, TABLE_UID(pTable)); - buf = taosEncodeFixedI32(buf, TABLE_TID(pTable)); + tlen = taosEncodeFixedU8(buf, pTable->type); + tlen = tsdbEncodeTableName(buf, pTable->name); + tlen = taosEncodeFixedU64(buf, TABLE_UID(pTable)); + tlen = taosEncodeFixedI32(buf, TABLE_TID(pTable)); if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { - buf = taosEncodeFixedU64(buf, TABLE_SUID(pTable)); - buf = tdEncodeKVRow(buf, pTable->tagVal); + tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable)); + tlen += tdEncodeKVRow(buf, pTable->tagVal); } else { - buf = taosEncodeFixedU8(buf, pTable->numOfSchemas); + tlen += taosEncodeFixedU8(buf, pTable->numOfSchemas); for (int i = 0; i < pTable->numOfSchemas; i++) { - buf = tdEncodeSchema(buf, pTable->schema[i]); + tlen += tdEncodeSchema(buf, pTable->schema[i]); } if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - buf = tdEncodeSchema(buf, pTable->tagSchema); + tlen += tdEncodeSchema(buf, pTable->tagSchema); } if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { - buf = taosEncodeString(buf, pTable->sql); + tlen += taosEncodeString(buf, pTable->sql); } } - return buf; + return tlen; } static void *tsdbDecodeTable(void *buf, STable **pRTable) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 28f562973f..ac3358cf2a 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -50,7 +50,7 @@ static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBl static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, int maxPoints, char *buffer, int bufferSize); static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols); -static void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx); +static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx); static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); static void tsdbDestroyHelperBlock(SRWHelper *pHelper); @@ -371,8 +371,8 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer) * 2); } buf = POINTER_SHIFT(pHelper->pBuffer, drift); - buf = taosEncodeVariantU32(buf, i); - buf = tsdbEncodeSCompIdx(buf, pCompIdx); + taosEncodeVariantU32(&buf, i); + tsdbEncodeSCompIdx(&buf, pCompIdx); } } @@ -1298,15 +1298,17 @@ _err: return -1; } -static void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) { - buf = taosEncodeVariantU32(buf, pIdx->len); - buf = taosEncodeVariantU32(buf, pIdx->offset); - buf = taosEncodeFixedU8(buf, pIdx->hasLast); - buf = taosEncodeVariantU32(buf, pIdx->numOfBlocks); - buf = taosEncodeFixedU64(buf, pIdx->uid); - buf = taosEncodeFixedU64(buf, pIdx->maxKey); +static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) { + int tlen = 0; - return buf; + tlen += taosEncodeVariantU32(buf, pIdx->len); + tlen += taosEncodeVariantU32(buf, pIdx->offset); + tlen += taosEncodeFixedU8(buf, pIdx->hasLast); + tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks); + tlen += taosEncodeFixedU64(buf, pIdx->uid); + tlen += taosEncodeFixedU64(buf, pIdx->maxKey); + + return tlen; } static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { diff --git a/src/util/inc/tcoding.h b/src/util/inc/tcoding.h index e22c959a56..907c69bf01 100644 --- a/src/util/inc/tcoding.h +++ b/src/util/inc/tcoding.h @@ -33,9 +33,13 @@ static const int32_t TNUMBER = 1; #define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode // ---- Fixed U8 -static FORCE_INLINE void *taosEncodeFixedU8(void *buf, uint8_t value) { - ((uint8_t *)buf)[0] = value; - return POINTER_SHIFT(buf, sizeof(value)); +static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) { + if (buf != NULL) { + ((uint8_t *)(*buf))[0] = value; + *buf = POINTER_SHIFT(*buf, sizeof(value)); + } + + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) { @@ -44,9 +48,12 @@ static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) { } // ---- Fixed I8 -static FORCE_INLINE void *taosEncodeFixedI8(void *buf, int8_t value) { - ((int8_t *)buf)[0] = value; - return POINTER_SHIFT(buf, sizeof(value)); +static FORCE_INLINE int taosEncodeFixedI8(void **buf, int8_t value) { + if (buf != NULL) { + ((int8_t *)(*buf))[0] = value; + *buf = POINTER_SHIFT(*buf, sizeof(value)); + } + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) { @@ -55,15 +62,18 @@ static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) { } // ---- Fixed U16 -static FORCE_INLINE void *taosEncodeFixedU16(void *buf, uint16_t value) { - if (IS_LITTLE_ENDIAN()) { - memcpy(buf, &value, sizeof(value)); - } else { - ((uint8_t *)buf)[0] = value & 0xff; - ((uint8_t *)buf)[1] = (value >> 8) & 0xff; +static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) { + if (buf != NULL) { + if (IS_LITTLE_ENDIAN()) { + memcpy(*buf, &value, sizeof(value)); + } else { + ((uint8_t *)(*buf))[0] = value & 0xff; + ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; + } + *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return POINTER_SHIFT(buf, sizeof(value)); + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) { @@ -78,7 +88,7 @@ static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) { } // ---- Fixed I16 -static FORCE_INLINE void *taosEncodeFixedI16(void *buf, int16_t value) { +static FORCE_INLINE int taosEncodeFixedI16(void **buf, int16_t value) { return taosEncodeFixedU16(buf, ZIGZAGE(int16_t, value)); } @@ -90,17 +100,20 @@ static FORCE_INLINE void *taosDecodeFixedI16(void *buf, int16_t *value) { } // ---- Fixed U32 -static FORCE_INLINE void *taosEncodeFixedU32(void *buf, uint32_t value) { - if (IS_LITTLE_ENDIAN()) { - memcpy(buf, &value, sizeof(value)); - } else { - ((uint8_t *)buf)[0] = value & 0xff; - ((uint8_t *)buf)[1] = (value >> 8) & 0xff; - ((uint8_t *)buf)[2] = (value >> 16) & 0xff; - ((uint8_t *)buf)[3] = (value >> 24) & 0xff; +static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) { + if (buf != NULL) { + if (IS_LITTLE_ENDIAN()) { + memcpy(*buf, &value, sizeof(value)); + } else { + ((uint8_t *)(*buf))[0] = value & 0xff; + ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; + ((uint8_t *)(*buf))[2] = (value >> 16) & 0xff; + ((uint8_t *)(*buf))[3] = (value >> 24) & 0xff; + } + *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return POINTER_SHIFT(buf, sizeof(value)); + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) { @@ -117,7 +130,7 @@ static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) { } // ---- Fixed I32 -static FORCE_INLINE void *taosEncodeFixedI32(void *buf, int32_t value) { +static FORCE_INLINE int taosEncodeFixedI32(void **buf, int32_t value) { return taosEncodeFixedU32(buf, ZIGZAGE(int32_t, value)); } @@ -129,21 +142,25 @@ static FORCE_INLINE void *taosDecodeFixedI32(void *buf, int32_t *value) { } // ---- Fixed U64 -static FORCE_INLINE void *taosEncodeFixedU64(void *buf, uint64_t value) { - if (IS_LITTLE_ENDIAN()) { - memcpy(buf, &value, sizeof(value)); - } else { - ((uint8_t *)buf)[0] = value & 0xff; - ((uint8_t *)buf)[1] = (value >> 8) & 0xff; - ((uint8_t *)buf)[2] = (value >> 16) & 0xff; - ((uint8_t *)buf)[3] = (value >> 24) & 0xff; - ((uint8_t *)buf)[4] = (value >> 32) & 0xff; - ((uint8_t *)buf)[5] = (value >> 40) & 0xff; - ((uint8_t *)buf)[6] = (value >> 48) & 0xff; - ((uint8_t *)buf)[7] = (value >> 56) & 0xff; +static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) { + if (buf != NULL) { + if (IS_LITTLE_ENDIAN()) { + memcpy(*buf, &value, sizeof(value)); + } else { + ((uint8_t *)(*buf))[0] = value & 0xff; + ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; + ((uint8_t *)(*buf))[2] = (value >> 16) & 0xff; + ((uint8_t *)(*buf))[3] = (value >> 24) & 0xff; + ((uint8_t *)(*buf))[4] = (value >> 32) & 0xff; + ((uint8_t *)(*buf))[5] = (value >> 40) & 0xff; + ((uint8_t *)(*buf))[6] = (value >> 48) & 0xff; + ((uint8_t *)(*buf))[7] = (value >> 56) & 0xff; + } + + *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return POINTER_SHIFT(buf, sizeof(value)); + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) { @@ -164,7 +181,7 @@ static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) { } // ---- Fixed I64 -static FORCE_INLINE void *taosEncodeFixedI64(void *buf, int64_t value) { +static FORCE_INLINE int taosEncodeFixedI64(void **buf, int64_t value) { return taosEncodeFixedU64(buf, ZIGZAGE(int64_t, value)); } @@ -176,18 +193,21 @@ static FORCE_INLINE void *taosDecodeFixedI64(void *buf, int64_t *value) { } // ---- Variant U16 -static FORCE_INLINE void *taosEncodeVariantU16(void *buf, uint16_t value) { +static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) { int i = 0; while (value >= ENCODE_LIMIT) { - ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; i++; ASSERT(i < 3); } - ((uint8_t *)buf)[i] = value; + if (buf != NULL) { + ((uint8_t *)(*buf))[i] = value; + *buf = POINTER_SHIFT(*buf, i + 1); + } - return POINTER_SHIFT(buf, i + 1); + return i + 1; } static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) { @@ -209,7 +229,7 @@ static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) { } // ---- Variant I16 -static FORCE_INLINE void *taosEncodeVariantI16(void *buf, int16_t value) { +static FORCE_INLINE int taosEncodeVariantI16(void **buf, int16_t value) { return taosEncodeVariantU16(buf, ZIGZAGE(int16_t, value)); } @@ -221,18 +241,21 @@ static FORCE_INLINE void *taosDecodeVariantI16(void *buf, int16_t *value) { } // ---- Variant U32 -static FORCE_INLINE void *taosEncodeVariantU32(void *buf, uint32_t value) { +static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) { int i = 0; while (value >= ENCODE_LIMIT) { - ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; i++; ASSERT(i < 5); } - ((uint8_t *)buf)[i] = value; + if (buf != NULL) { + ((uint8_t *)(*buf))[i] = value; + *buf = POINTER_SHIFT(*buf, i + 1); + } - return POINTER_SHIFT(buf, i + 1); + return i + 1; } static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) { @@ -254,7 +277,7 @@ static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) { } // ---- Variant I32 -static FORCE_INLINE void *taosEncodeVariantI32(void *buf, int32_t value) { +static FORCE_INLINE int taosEncodeVariantI32(void **buf, int32_t value) { return taosEncodeVariantU32(buf, ZIGZAGE(int32_t, value)); } @@ -266,18 +289,21 @@ static FORCE_INLINE void *taosDecodeVariantI32(void *buf, int32_t *value) { } // ---- Variant U64 -static FORCE_INLINE void *taosEncodeVariantU64(void *buf, uint64_t value) { +static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) { int i = 0; while (value >= ENCODE_LIMIT) { - ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; i++; ASSERT(i < 10); } - ((uint8_t *)buf)[i] = value; + if (buf != NULL) { + ((uint8_t *)(*buf))[i] = value; + *buf = POINTER_SHIFT(*buf, i + 1); + } - return POINTER_SHIFT(buf, i + 1); + return i + 1; } static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) { @@ -299,7 +325,7 @@ static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) { } // ---- Variant I64 -static FORCE_INLINE void *taosEncodeVariantI64(void *buf, int64_t value) { +static FORCE_INLINE int taosEncodeVariantI64(void **buf, int64_t value) { return taosEncodeVariantU64(buf, ZIGZAGE(int64_t, value)); } @@ -311,13 +337,18 @@ static FORCE_INLINE void *taosDecodeVariantI64(void *buf, int64_t *value) { } // ---- string -static FORCE_INLINE void *taosEncodeString(void *buf, char *value) { +static FORCE_INLINE int taosEncodeString(void **buf, char *value) { + int tlen = 0; size_t size = strlen(value); - buf = taosEncodeVariantU64(buf, size); - memcpy(buf, value, size); + tlen += taosEncodeVariantU64(buf, size); + if (buf != NULL) { + memcpy(*buf, value, size); + *buf = POINTER_SHIFT(*buf, size); + } + tlen += size; - return POINTER_SHIFT(buf, size); + return tlen; } static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index be9c4c19b0..a84a5ac011 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -41,7 +41,7 @@ typedef struct { } SKVRecord; static int tdInitKVStoreHeader(int fd, char *fname); -static void * tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo); +static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo); static void * tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo); static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); static char * tdGetKVStoreSnapshotFname(char *fdata); @@ -49,7 +49,7 @@ static char * tdGetKVStoreNewFname(char *fdata); static void tdFreeKVStore(SKVStore *pStore); static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo); static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo); -static void * tdEncodeKVRecord(void *buf, SKVRecord *pRecord); +static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord); static void * tdDecodeKVRecord(void *buf, SKVRecord *pRecord); static int tdRestoreKVStore(SKVStore *pStore); @@ -269,7 +269,8 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { rInfo.uid = pRecord->uid; rInfo.size = pRecord->size; - void *pBuf = tdEncodeKVRecord(buf, &rInfo); + void *pBuf = buf; + tdEncodeKVRecord(&pBuf, &rInfo); if (twrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) { uError("failed to write %d bytes to file %s since %s", POINTER_DISTANCE(pBuf, buf), pStore->fname, strerror(errno)); @@ -344,7 +345,8 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { return -1; } - void *pBuf = tdEncodeStoreInfo(buf, pInfo); + void *pBuf = buf; + tdEncodeStoreInfo(&pBuf, pInfo); ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE); taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE); @@ -363,13 +365,14 @@ static int tdInitKVStoreHeader(int fd, char *fname) { return tdUpdateKVStoreHeader(fd, fname, &info); } -static void *tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo) { - buf = taosEncodeVariantI64(buf, pInfo->size); - buf = taosEncodeVariantI64(buf, pInfo->tombSize); - buf = taosEncodeVariantI64(buf, pInfo->nRecords); - buf = taosEncodeVariantI64(buf, pInfo->nDels); +static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) { + int tlen = 0; + tlen += taosEncodeVariantI64(buf, pInfo->size); + tlen += taosEncodeVariantI64(buf, pInfo->tombSize); + tlen += taosEncodeVariantI64(buf, pInfo->nRecords); + tlen += taosEncodeVariantI64(buf, pInfo->nDels); - return buf; + return tlen; } static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) { @@ -450,12 +453,13 @@ static char *tdGetKVStoreNewFname(char *fdata) { return fname; } -static void *tdEncodeKVRecord(void *buf, SKVRecord *pRecord) { - buf = taosEncodeFixedU64(buf, pRecord->uid); - buf = taosEncodeFixedI64(buf, pRecord->offset); - buf = taosEncodeFixedI64(buf, pRecord->size); +static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pRecord->uid); + tlen += taosEncodeFixedI64(buf, pRecord->offset); + tlen += taosEncodeFixedI64(buf, pRecord->size); - return buf; + return tlen; } static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) { diff --git a/src/util/tests/codingTests.cpp b/src/util/tests/codingTests.cpp index 57e21a828c..0cd9524646 100644 --- a/src/util/tests/codingTests.cpp +++ b/src/util/tests/codingTests.cpp @@ -9,120 +9,144 @@ static bool test_fixed_uint16(uint16_t value) { char buf[20] = "\0"; uint16_t value_check = 0; - void *ptr1 = taosEncodeFixedU16(static_cast(buf), value); - void *ptr2 = taosDecodeFixedU16(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeFixedU16(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedU16(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_int16(int16_t value) { char buf[20] = "\0"; int16_t value_check = 0; - void *ptr1 = taosEncodeFixedI16(static_cast(buf), value); - void *ptr2 = taosDecodeFixedI16(static_cast(buf), &value_check); + void *pBuf = (void *)buf; + + int tlen = taosEncodeFixedI16(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedI16(static_cast(buf), &value_check); - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_uint32(uint32_t value) { char buf[20] = "\0"; uint32_t value_check = 0; - void *ptr1 = taosEncodeFixedU32(static_cast(buf), value); - void *ptr2 = taosDecodeFixedU32(static_cast(buf), &value_check); + void *pBuf = (void *)buf; + + int tlen = taosEncodeFixedU32(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedU32(static_cast(buf), &value_check); - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_int32(int32_t value) { char buf[20] = "\0"; int32_t value_check = 0; - void *ptr1 = taosEncodeFixedI32(static_cast(buf), value); - void *ptr2 = taosDecodeFixedI32(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeFixedI32(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedI32(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_uint64(uint64_t value) { char buf[20] = "\0"; uint64_t value_check = 0; - void *ptr1 = taosEncodeFixedU64(static_cast(buf), value); - void *ptr2 = taosDecodeFixedU64(static_cast(buf), &value_check); + void *pBuf = (void *)buf; + + int tlen = taosEncodeFixedU64(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedU64(static_cast(buf), &value_check); - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_int64(int64_t value) { char buf[20] = "\0"; int64_t value_check = 0; - void *ptr1 = taosEncodeFixedI64(static_cast(buf), value); - void *ptr2 = taosDecodeFixedI64(static_cast(buf), &value_check); + void *pBuf = (void *)buf; + + int tlen = taosEncodeFixedI64(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedI64(static_cast(buf), &value_check); - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_uint16(uint16_t value) { char buf[20] = "\0"; uint16_t value_check = 0; - void *ptr1 = taosEncodeVariantU16(static_cast(buf), value); - void *ptr2 = taosDecodeVariantU16(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeVariantU16(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantU16(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_int16(int16_t value) { char buf[20] = "\0"; int16_t value_check = 0; - void *ptr1 = taosEncodeVariantI16(static_cast(buf), value); - void *ptr2 = taosDecodeVariantI16(static_cast(buf), &value_check); + void *pBuf = (void *)buf; + + int tlen = taosEncodeVariantI16(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantI16(static_cast(buf), &value_check); - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_uint32(uint32_t value) { char buf[20] = "\0"; uint32_t value_check = 0; - void *ptr1 = taosEncodeVariantU32(static_cast(buf), value); - void *ptr2 = taosDecodeVariantU32(static_cast(buf), &value_check); + void *pBuf = (void *)buf; + + int tlen = taosEncodeVariantU32(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantU32(static_cast(buf), &value_check); - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_int32(int32_t value) { char buf[20] = "\0"; int32_t value_check = 0; - void *ptr1 = taosEncodeVariantI32(static_cast(buf), value); - void *ptr2 = taosDecodeVariantI32(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeVariantI32(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantI32(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_uint64(uint64_t value) { char buf[20] = "\0"; uint64_t value_check = 0; - void *ptr1 = taosEncodeVariantU64(static_cast(buf), value); - void *ptr2 = taosDecodeVariantU64(static_cast(buf), &value_check); + void *pBuf = (void *)buf; + + int tlen = taosEncodeVariantU64(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantU64(static_cast(buf), &value_check); - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_int64(int64_t value) { char buf[20] = "\0"; int64_t value_check = 0; - void *ptr1 = taosEncodeVariantI64(static_cast(buf), value); - void *ptr2 = taosDecodeVariantI64(static_cast(buf), &value_check); + void *pBuf = (void *)buf; + + int tlen = taosEncodeVariantI64(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantI64(static_cast(buf), &value_check); - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } TEST(codingTest, fixed_encode_decode) { -- GitLab