Merge pull request #5276 from taosdata/hotfix/TD-3075

[TD-3075]<hotfix>: fix SBlockCol offset overflow

src/tsdb/inc/tsdbReadImpl.h

@@ -49,19 +49,32 @@ typedef struct {
 } SBlockInfo;
 
 typedef struct {
-  int16_t colId;
-  int32_t len;
-  int32_t type : 8;
-  int32_t offset : 24;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-  char    padding[2];
+  int16_t  colId;
+  int32_t  len;
+  uint32_t type : 8;
+  uint32_t offset : 24;
+  int64_t  sum;
+  int64_t  max;
+  int64_t  min;
+  int16_t  maxIndex;
+  int16_t  minIndex;
+  int16_t  numOfNull;
+  uint8_t  offsetH;
+  char     padding[1];
 } SBlockCol;
 
+// Code here just for back-ward compatibility
+static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
+  pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
+  pBlockCol->offsetH = (uint8_t)(offset >> 24);
+}
+
+static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
+  uint32_t offset1 = pBlockCol->offset;
+  uint32_t offset2 = pBlockCol->offsetH;
+  return (offset1 | (offset2 << 24));
+}
+
 typedef struct {
   int32_t   delimiter;  // For recovery usage
   int32_t   numOfCols;  // For recovery usage

src/tsdb/src/tsdbCommit.c

@@ -781,11 +781,11 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo
   ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols);
 
   // Compress the data if neccessary
-  int     tcol = 0;  // counter of not all NULL and written columns
-  int32_t toffset = 0;
-  int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull);
-  int32_t lsize = tsize;
-  int32_t keyLen = 0;
+  int      tcol = 0;  // counter of not all NULL and written columns
+  uint32_t toffset = 0;
+  int32_t  tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull);
+  int32_t  lsize = tsize;
+  int32_t  keyLen = 0;
   for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
     // All not NULL columns finish
     if (ncol != 0 && tcol >= nColsNotAllNull) break;
@@ -829,7 +829,7 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo
     tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)));
 
     if (ncol != 0) {
-      pBlockCol->offset = toffset;
+      tsdbSetBlockColOffset(pBlockCol, toffset);
       pBlockCol->len = flen;
       tcol++;
     } else {

src/tsdb/src/tsdbReadImpl.c

@@ -466,14 +466,14 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       continue;
     }
 
-    int16_t tcolId = 0;
-    int32_t toffset = TSDB_KEY_COL_OFFSET;
-    int32_t tlen = pBlock->keyLen;
+    int16_t  tcolId = 0;
+    uint32_t toffset = TSDB_KEY_COL_OFFSET;
+    int32_t  tlen = pBlock->keyLen;
 
     if (dcol != 0) {
       SBlockCol *pBlockCol = &(pBlockData->cols[ccol]);
       tcolId = pBlockCol->colId;
-      toffset = pBlockCol->offset;
+      toffset = tsdbGetBlockColOffset(pBlockCol);
       tlen = pBlockCol->len;
     } else {
       ASSERT(pDataCol->colId == tcolId);
@@ -488,7 +488,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
                                        pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
                                        (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
-        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d",
+        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u",
                   TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
         return -1;
       }
@@ -627,7 +627,7 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
   if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
   if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
 
-  int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + pBlockCol->offset;
+  int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + tsdbGetBlockColOffset(pBlockCol);
   if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
     tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s",
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));