diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 6065feedea79e627da88c709823d36eab055b23c..3c5cb4f23a2dfd28e8c8be8255c350b751856fa5 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -359,39 +359,39 @@ typedef struct { SSubmitBlk* pSubmitBlk; } SMemRowBuilder; -int tdInitMemRowBuilder(SMemRowBuilder* pBuilder); -void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder); -void tdResetMemRowBuilder(SMemRowBuilder* pBuilder); -SMemRow tdGetMemRowFromBuilder(SMemRowBuilder* pBuilder); - -static FORCE_INLINE int tdAddColToMemRow(SMemRowBuilder* pBuilder, int16_t colId, int8_t type, void* value) { - // TODO - - if (pBuilder->nCols >= pBuilder->tCols) { - pBuilder->tCols *= 2; - pBuilder->pColIdx = (SColIdx*)realloc((void*)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); - if (pBuilder->pColIdx == NULL) return -1; - } - - pBuilder->pColIdx[pBuilder->nCols].colId = colId; - pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size; - - pBuilder->nCols++; - - int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; - if (tlen > pBuilder->alloc - pBuilder->size) { - while (tlen > pBuilder->alloc - pBuilder->size) { - pBuilder->alloc *= 2; - } - pBuilder->buf = realloc(pBuilder->buf, pBuilder->alloc); - if (pBuilder->buf == NULL) return -1; - } - - memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen); - pBuilder->size += tlen; - - return 0; -} +// int tdInitMemRowBuilder(SMemRowBuilder* pBuilder); +// void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder); +// void tdResetMemRowBuilder(SMemRowBuilder* pBuilder); +SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder); + +// static FORCE_INLINE int tdAddColToMemRow(SMemRowBuilder* pBuilder, int16_t colId, int8_t type, void* value) { +// // TODO + +// if (pBuilder->nCols >= pBuilder->tCols) { +// pBuilder->tCols *= 2; +// pBuilder->pColIdx = (SColIdx*)realloc((void*)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); +// if (pBuilder->pColIdx == NULL) return -1; +// } + +// pBuilder->pColIdx[pBuilder->nCols].colId = colId; +// pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size; + +// pBuilder->nCols++; + +// int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; +// if (tlen > pBuilder->alloc - pBuilder->size) { +// while (tlen > pBuilder->alloc - pBuilder->size) { +// pBuilder->alloc *= 2; +// } +// pBuilder->buf = realloc(pBuilder->buf, pBuilder->alloc); +// if (pBuilder->buf == NULL) return -1; +// } + +// memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen); +// pBuilder->size += tlen; + +// return 0; +// } #ifdef __cplusplus } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index bda309a15938c3274388b08db462b6a847735c82..2c53d90cea73a7d63734027e20f2c0e1bebc36c2 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -14,7 +14,7 @@ */ #include "tscUtil.h" -#include "hash.h" +#include "hash.h" #include "os.h" #include "taosmsg.h" #include "texpr.h" @@ -1640,35 +1640,34 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } -int tdInitMemRowBuilder(SMemRowBuilder* pBuilder) { - pBuilder->pSchema = NULL; - pBuilder->sversion = 0; - pBuilder->tCols = 128; - pBuilder->nCols = 0; - pBuilder->pColIdx = (SColIdx*)malloc(sizeof(SColIdx) * pBuilder->tCols); - if (pBuilder->pColIdx == NULL) return -1; - pBuilder->alloc = 1024; - pBuilder->size = 0; - pBuilder->buf = malloc(pBuilder->alloc); - if (pBuilder->buf == NULL) { - free(pBuilder->pColIdx); - return -1; - } - return 0; -} - -void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder) { - tfree(pBuilder->pColIdx); - tfree(pBuilder->buf); -} - -void tdResetMemRowBuilder(SMemRowBuilder* pBuilder) { - pBuilder->nCols = 0; - pBuilder->size = 0; -} - -#define KvRowNullColRatio 0.75 // If nullable column ratio larger than 0.75, utilize SKVRow, otherwise SDataRow. -#define KvRowNColsThresh 1 // default value: 32 +// int tdInitMemRowBuilder(SMemRowBuilder* pBuilder) { +// pBuilder->pSchema = NULL; +// pBuilder->sversion = 0; +// pBuilder->tCols = 128; +// pBuilder->nCols = 0; +// pBuilder->pColIdx = (SColIdx*)malloc(sizeof(SColIdx) * pBuilder->tCols); +// if (pBuilder->pColIdx == NULL) return -1; +// pBuilder->alloc = 1024; +// pBuilder->size = 0; +// pBuilder->buf = malloc(pBuilder->alloc); +// if (pBuilder->buf == NULL) { +// free(pBuilder->pColIdx); +// return -1; +// } +// return 0; +// } + +// void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder) { +// tfree(pBuilder->pColIdx); +// tfree(pBuilder->buf); +// } + +// void tdResetMemRowBuilder(SMemRowBuilder* pBuilder) { +// pBuilder->nCols = 0; +// pBuilder->size = 0; +// } + +#define KvRowNColsThresh 1 // default value: 32 TODO: for test, restore to 32 after test finished static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, uint16_t* nColsNotNull) { @@ -1701,7 +1700,8 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 p += pSchema[i].bytes; } - tscInfo("prop:nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", nColsNull, nCols, kvRowLength, dataRowLength); + tscDebug("prop:nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", (int32_t)nColsNull, nCols, kvRowLength, + dataRowLength); if (kvRowLength < dataRowLength) { if (nColsNotNull) { @@ -1713,7 +1713,7 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 return SMEM_ROW_DATA; } -SMemRow tdGetMemRowFromBuilder(SMemRowBuilder* pBuilder) { +SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { SSchema* pSchema = pBuilder->pSchema; char* p = (char*)pBuilder->buf; @@ -1840,7 +1840,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo pDataBlock = (char*)pDataBlock + dataRowLen(trow); // next SDataRow pBlock->dataLen += dataRowLen(trow); // SSubmitBlk data length #endif - tdGetMemRowFromBuilder(&mRowBuilder); + tdGenMemRowFromBuilder(&mRowBuilder); } int32_t len = pBlock->dataLen + pBlock->schemaLen; diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 4b69192b7be678af371aa5ae2182f2e3206245c3..5c3ccdd781cc45e9d5aa985ea9812a5ae40ac2e2 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -224,10 +224,10 @@ typedef void *SMemRow; #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) #define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) -// SDataRow tdNewDataRowFromSchema(STSchema *pSchema); -// void tdFreeDataRow(SDataRow row); +SDataRow tdNewDataRowFromSchema(STSchema *pSchema); +void tdFreeDataRow(SDataRow row); void tdInitDataRow(SDataRow row, STSchema *pSchema); -// SDataRow tdDataRowDup(SDataRow row); +SDataRow tdDataRowDup(SDataRow row); SMemRow tdMemRowDup(SMemRow row); // offset here not include dataRow header length @@ -253,6 +253,16 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t return 0; } +// NOTE: offset here including the header size +static FORCE_INLINE void *tdGetRowDataOfCol(void *row, int8_t type, int32_t offset) { + if (IS_VAR_DATA_TYPE(type)) { + return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); + } else { + return POINTER_SHIFT(row, offset); + } + return NULL; +} + // ----------------- Data column structure typedef struct SDataCol { int8_t type; // column type @@ -519,27 +529,26 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, return 0; } -// ----------------- Data row structure - // ----------------- Sequential Data row structure -/* A sequential data row, the format is like below: - * |<--------------------+--------------------------- len ---------------------------------->| - * |<-- Head -->|<--------- flen -------------->| | - * +---------------------+---------------------------------+---------------------------------+ - * | uint16_t | int16_t | | | - * +----------+----------+---------------------------------+---------------------------------+ - * | len | sversion | First part | Second part | - * +----------+----------+---------------------------------+---------------------------------+ +/* + * |-------------------------------+--------------------------- len ---------------------------------->| + * |<-------- Head ------>|<--------- flen -------------->| | + * |---------+---------------------+---------------------------------+---------------------------------+ + * | uint8_t | uint16_t | int16_t | | | + * |---------+----------+----------+---------------------------------+---------------------------------+ + * | flag | len | sversion | First part | Second part | + * +---------+----------+----------+---------------------------------+---------------------------------+ * * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ + // ----------------- K-V data row structure /* - * +----------+----------+---------------------------------+---------------------------------+ - * | int16_t | int16_t | | | - * +----------+----------+---------------------------------+---------------------------------+ - * | len | ncols | cols index | data part | - * +----------+----------+---------------------------------+---------------------------------+ + * |--------------------+----------+---------------------------------+---------------------------------+ + * | uint8_t | uint16_t | int16_t | | | + * |---------+----------+----------+---------------------------------+---------------------------------+ + * | flag | len | ncols | cols index | data part | + * |---------+----------+----------+---------------------------------+---------------------------------+ */ #define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) @@ -572,24 +581,11 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, #define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_HEAD_SIZE) #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) -// #define memRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE + sizeof(int16_t))) // for SKVRow -// #define memRowSetNCols(r, n) memRowNCols(r) = (n) // for SKVRow -// #define memRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE) // for SKVRow -// #define memRowValues(r) POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE + sizeof(SColIdx) * memRowNCols(r)) // for SKVRow - // NOTE: offset here including the header size -static FORCE_INLINE void *tdGetRowDataOfCol(void *row, int8_t type, int32_t offset) { - if (IS_VAR_DATA_TYPE(type)) { - return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - } else { - return POINTER_SHIFT(row, offset); - } - return NULL; -} static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int8_t type, int32_t offset) { return POINTER_SHIFT(row, offset); } - +// NOTE: offset here including the header size static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t offset) { if (isDataRow(row)) { return tdGetRowDataOfCol(row, type, offset); @@ -601,12 +597,6 @@ static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t o return NULL; } -// #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) -// #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) -// #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -// #define kvRowFree(r) tfree(r) -// #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) - #ifdef __cplusplus } #endif diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index f962e5255bc1d9e052d606bdec1f39b4eab6c3c7..ac7b15ea025f0c2d3699ab40f97915e97f030ec7 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -188,30 +188,30 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema) { dataRowSetVersion(row, schemaVersion(pSchema)); } -// SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { -// int32_t size = dataRowMaxBytesFromSchema(pSchema); +SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { + int32_t size = dataRowMaxBytesFromSchema(pSchema); -// SDataRow row = malloc(size); -// if (row == NULL) return NULL; + SDataRow row = malloc(size); + if (row == NULL) return NULL; -// tdInitDataRow(row, pSchema); -// return row; -// } + tdInitDataRow(row, pSchema); + return row; +} /** * Free the SDataRow object */ -// void tdFreeDataRow(SDataRow row) { -// if (row) free(row); -// } +void tdFreeDataRow(SDataRow row) { + if (row) free(row); +} -// SDataRow tdDataRowDup(SDataRow row) { -// SDataRow trow = malloc(dataRowLen(row)); -// if (trow == NULL) return NULL; +SDataRow tdDataRowDup(SDataRow row) { + SDataRow trow = malloc(dataRowLen(row)); + if (trow == NULL) return NULL; -// dataRowCpy(trow, row); -// return trow; -// } + dataRowCpy(trow, row); + return trow; +} SMemRow tdMemRowDup(SMemRow row) { SMemRow trow = malloc(memRowTLen(row)); @@ -220,6 +220,7 @@ SMemRow tdMemRowDup(SMemRow row) { memRowCpy(trow, row); return trow; } + void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) { pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); @@ -436,7 +437,7 @@ void tdResetDataCols(SDataCols *pCols) { static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); - int rcol = 0; // rowCol + int rcol = 0; int dcol = 0; if (dataRowDeleted(row)) { diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 03083795092f1bcbf3f4859a0ac12b00b9b49740..80b8ddcd4eed563b803e54cdccc5812d3a4e582a 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -11,7 +11,7 @@ extern "C" { // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; // maxDataLen: 32767 +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 typedef struct tstr { VarDataLenT len; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 0c87a7b735888d3bd63a05b821ccc8aa03a7147b..faf099cdd21d19a88d2d4ae36491f86b65e48970 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1442,107 +1442,6 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity return numOfRows + num; } -#if 0 -static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, - int32_t numOfCols, STable* pTable, STSchema* pSchema) { - char* pData = NULL; - - // the schema version info is embedded in SDataRow, and use latest schema version for SKVRow - int32_t numOfRowCols = 0; - if (pSchema == NULL) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); - numOfRowCols = schemaNCols(pSchema); - } else { - numOfRowCols = schemaNCols(pSchema); - } - - int32_t i = 0, j = 0; - while(i < numOfCols && j < numOfRowCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (pSchema->columns[j].colId < pColInfo->info.colId) { - j++; - continue; - } - - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } - - if (pSchema->columns[j].colId == pColInfo->info.colId) { - void* value = - tdGetMemRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_MEM_ROW_HEAD_SIZE + pSchema->columns[j].offset); - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY *)pData = tdGetKey(*(TKEY *)value); - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - - j++; - i++; - } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - i++; - } - } - - while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - - i++; - } -} -#endif static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, int32_t numOfCols, STable* pTable, STSchema* pSchema) {