From 85343a0eb9994e7c7c4b0ca65ec9fd3fa3b689e8 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 11 Oct 2021 20:35:39 +0800 Subject: [PATCH] [TS-414]: pCol->len logic fix for partial update of columns --- src/common/inc/tdataformat.h | 12 ++++++++++-- src/common/src/tdataformat.c | 37 +++++++++++++++++++++--------------- src/tsdb/src/tsdbCommit.c | 5 +++-- src/tsdb/src/tsdbMemTable.c | 2 +- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index a01c377539..426c3e7ec1 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -339,7 +339,15 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } int tdAllocMemForCol(SDataCol *pCol, int maxPoints); void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints); -int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); + +// value from timestamp should be TKEY here instead of TSKEY +int dataColAppendValEx(SDataCol *pCol, const void *value, int numOfRows, int maxPoints, bool isAppend); + +// value from timestamp should be TKEY here instead of TSKEY +FORCE_INLINE int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { + return dataColAppendValEx(pCol, value, numOfRows, maxPoints, true); +} + void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); @@ -670,7 +678,7 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) { #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) SMemRow tdMemRowDup(SMemRow row); -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull); +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull, bool isAppend); // NOTE: offset here including the header size static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) { diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 731dfecfb6..e773a06900 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -240,7 +240,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->len = 0; } // value from timestamp should be TKEY here instead of TSKEY -int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { +int dataColAppendValEx(SDataCol *pCol, const void *value, int numOfRows, int maxPoints, bool isAppend) { ASSERT(pCol != NULL && value != NULL); if (isAllRowsNull(pCol)) { @@ -264,9 +264,15 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo // Update the length pCol->len += varDataTLen(value); } else { - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); - pCol->len += pCol->bytes; + if (isAppend) { // append the value + ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); + pCol->len += pCol->bytes; + } else { + // update the value of last row by increasing the pCol->len and keeping the original numOfRows + ASSERT((pCol->len == TYPE_BYTES[pCol->type] * (numOfRows + 1)) && (pCol->len > TYPE_BYTES[pCol->type])); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), value, pCol->bytes); + } } return 0; } @@ -441,7 +447,8 @@ void tdResetDataCols(SDataCols *pCols) { } } -static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { +static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull, + bool isAppend) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); int rcol = 0; @@ -451,7 +458,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols bool setCol = 0; SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendValEx(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, isAppend); dcol++; continue; } @@ -460,14 +467,14 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols if (pRowCol->colId == pDataCol->colId) { void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE); if(!isNull(value, pDataCol->type)) setCol = 1; - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); + dataColAppendValEx(pDataCol, value, pCols->numOfRows, pCols->maxPoints, isAppend); dcol++; rcol++; } else if (pRowCol->colId < pDataCol->colId) { rcol++; } else { if(forceSetNull || setCol) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendValEx(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, isAppend); } dcol++; } @@ -475,7 +482,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols pCols->numOfRows++; } -static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { +static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull, bool isAppend) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row)); int rcol = 0; @@ -487,7 +494,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo bool setCol = 0; SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendValEx(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, isAppend); ++dcol; continue; } @@ -497,14 +504,14 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo if (colIdx->colId == pDataCol->colId) { void *value = tdGetKvRowDataOfCol(row, colIdx->offset); if(!isNull(value, pDataCol->type)) setCol = 1; - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); + dataColAppendValEx(pDataCol, value, pCols->numOfRows, pCols->maxPoints, isAppend); ++dcol; ++rcol; } else if (colIdx->colId < pDataCol->colId) { ++rcol; } else { if (forceSetNull || setCol) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendValEx(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, isAppend); } ++dcol; } @@ -512,11 +519,11 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo pCols->numOfRows++; } -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull, bool isAppend) { if (isDataRow(row)) { - tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull); + tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull, isAppend); } else if (isKvRow(row)) { - tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull); + tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull, isAppend); } else { ASSERT(0); } diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 2c36e1b67a..a736657183 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -1288,7 +1288,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendMemRowToDataCol(row, pSchema, pTarget, true); + tdAppendMemRowToDataCol(row, pSchema, pTarget, true, true); tSkipListIterNext(pCommitIter->pIter); } else { @@ -1310,7 +1310,8 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); + tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE, + update != TD_ROW_PARTIAL_UPDATE); } (*iter)++; tSkipListIterNext(pCommitIter->pIter); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index c82490c3e8..5b363808f2 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -589,7 +589,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * } } - tdAppendMemRowToDataCol(row, *ppSchema, pCols, true); + tdAppendMemRowToDataCol(row, *ppSchema, pCols, true, true); } return 0; -- GitLab