[TS-414]<fix>: pCol->len logic fix for partial update of columns

src/common/inc/tdataformat.h

@@ -339,7 +339,15 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 int tdAllocMemForCol(SDataCol *pCol, int maxPoints);
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints);
-int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
+
+// value from timestamp should be TKEY here instead of TSKEY
+int dataColAppendValEx(SDataCol *pCol, const void *value, int numOfRows, int maxPoints, bool isAppend);
+
+// value from timestamp should be TKEY here instead of TSKEY
+FORCE_INLINE int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
+  return dataColAppendValEx(pCol, value, numOfRows, maxPoints, true);
+}
+
 void dataColSetOffset(SDataCol *pCol, int nEle);
 
 bool isNEleNull(SDataCol *pCol, int nEle);
@@ -670,7 +678,7 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) {
 #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r))
 
 SMemRow tdMemRowDup(SMemRow row);
-void    tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull);
+void    tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull, bool isAppend);
 
 // NOTE: offset here including the header size
 static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) {

src/common/src/tdataformat.c

@@ -240,7 +240,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
   pDataCol->len = 0;
 }
 // value from timestamp should be TKEY here instead of TSKEY
-int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
+int dataColAppendValEx(SDataCol *pCol, const void *value, int numOfRows, int maxPoints, bool isAppend) {
   ASSERT(pCol != NULL && value != NULL);
 
   if (isAllRowsNull(pCol)) {
@@ -264,9 +264,15 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo
     // Update the length
     pCol->len += varDataTLen(value);
   } else {
-    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
-    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
-    pCol->len += pCol->bytes;
+    if (isAppend) {  // append the value
+      ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
+      memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
+      pCol->len += pCol->bytes;
+    } else {
+      // update the value of last row by increasing the pCol->len and keeping the original numOfRows
+      ASSERT((pCol->len == TYPE_BYTES[pCol->type] * (numOfRows + 1)) && (pCol->len > TYPE_BYTES[pCol->type]));
+      memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), value, pCol->bytes);
+    }
   }
   return 0;
 }
@@ -441,7 +447,8 @@ void tdResetDataCols(SDataCols *pCols) {
   }
 }
 
-static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
+static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull,
+                                     bool isAppend) {
   ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
 
   int rcol = 0;
@@ -451,7 +458,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols
     bool setCol = 0;
     SDataCol *pDataCol = &(pCols->cols[dcol]);
     if (rcol >= schemaNCols(pSchema)) {
-      dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
+      dataColAppendValEx(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, isAppend);
       dcol++;
       continue;
     }
@@ -460,14 +467,14 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols
     if (pRowCol->colId == pDataCol->colId) {
       void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
       if(!isNull(value, pDataCol->type)) setCol = 1;
-      dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
+      dataColAppendValEx(pDataCol, value, pCols->numOfRows, pCols->maxPoints, isAppend);
       dcol++;
       rcol++;
     } else if (pRowCol->colId < pDataCol->colId) {
       rcol++;
     } else {
       if(forceSetNull || setCol) {
-        dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
+        dataColAppendValEx(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, isAppend);
       }
       dcol++;
     }
@@ -475,7 +482,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols
   pCols->numOfRows++;
 }
 
-static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
+static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull, bool isAppend) {
   ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row));
 
   int rcol = 0;
@@ -487,7 +494,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo
     bool setCol = 0;
     SDataCol *pDataCol = &(pCols->cols[dcol]);
     if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) {
-      dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
+      dataColAppendValEx(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, isAppend);
       ++dcol;
       continue;
     }
@@ -497,14 +504,14 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo
     if (colIdx->colId == pDataCol->colId) {
       void *value = tdGetKvRowDataOfCol(row, colIdx->offset);
       if(!isNull(value, pDataCol->type)) setCol = 1;
-      dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
+      dataColAppendValEx(pDataCol, value, pCols->numOfRows, pCols->maxPoints, isAppend);
       ++dcol;
       ++rcol;
     } else if (colIdx->colId < pDataCol->colId) {
       ++rcol;
     } else {
       if (forceSetNull || setCol) {
-        dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
+        dataColAppendValEx(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, isAppend);
       }
       ++dcol;
     }
@@ -512,11 +519,11 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo
   pCols->numOfRows++;
 }
 
-void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
+void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull, bool isAppend) {
   if (isDataRow(row)) {
-    tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull);
+    tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull, isAppend);
   } else if (isKvRow(row)) {
-    tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull);
+    tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull, isAppend);
   } else {
     ASSERT(0);
   }

src/tsdb/src/tsdbCommit.c

@@ -1288,7 +1288,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
         ASSERT(pSchema != NULL);
       }
 
-      tdAppendMemRowToDataCol(row, pSchema, pTarget, true);
+      tdAppendMemRowToDataCol(row, pSchema, pTarget, true, true);
 
       tSkipListIterNext(pCommitIter->pIter);
     } else {
@@ -1310,7 +1310,8 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
           ASSERT(pSchema != NULL);
         }
 
-        tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE);
+        tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE,
+                                update != TD_ROW_PARTIAL_UPDATE);
       }
       (*iter)++;
       tSkipListIterNext(pCommitIter->pIter);

src/tsdb/src/tsdbMemTable.c

@@ -589,7 +589,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema *
       }
     }
 
-    tdAppendMemRowToDataCol(row, *ppSchema, pCols, true);
+    tdAppendMemRowToDataCol(row, *ppSchema, pCols, true, true);
   }
 
   return 0;