Merge pull request #5276 from taosdata/hotfix/TD-3075

[TD-3075]<hotfix>: fix SBlockCol offset overflow

src/tsdb/inc/tsdbReadImpl.h

@@ -51,17 +51,30 @@ typedef struct {
 typedef struct {
   int16_t  colId;
   int32_t  len;
-  int32_t type : 8;
-  int32_t offset : 24;
+  uint32_t type : 8;
+  uint32_t offset : 24;
   int64_t  sum;
   int64_t  max;
   int64_t  min;
   int16_t  maxIndex;
   int16_t  minIndex;
   int16_t  numOfNull;
-  char    padding[2];
+  uint8_t  offsetH;
+  char     padding[1];
 } SBlockCol;
 
+// Code here just for back-ward compatibility
+static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
+  pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
+  pBlockCol->offsetH = (uint8_t)(offset >> 24);
+}
+
+static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
+  uint32_t offset1 = pBlockCol->offset;
+  uint32_t offset2 = pBlockCol->offsetH;
+  return (offset1 | (offset2 << 24));
+}
+
 typedef struct {
   int32_t   delimiter;  // For recovery usage
   int32_t   numOfCols;  // For recovery usage

src/tsdb/src/tsdbCommit.c

@@ -782,7 +782,7 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo
 
   // Compress the data if neccessary
   int      tcol = 0;  // counter of not all NULL and written columns
-  int32_t toffset = 0;
+  uint32_t toffset = 0;
   int32_t  tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull);
   int32_t  lsize = tsize;
   int32_t  keyLen = 0;
@@ -829,7 +829,7 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo
     tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)));
 
     if (ncol != 0) {
-      pBlockCol->offset = toffset;
+      tsdbSetBlockColOffset(pBlockCol, toffset);
       pBlockCol->len = flen;
       tcol++;
     } else {

src/tsdb/src/tsdbReadImpl.c

@@ -467,13 +467,13 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
     }
 
     int16_t  tcolId = 0;
-    int32_t toffset = TSDB_KEY_COL_OFFSET;
+    uint32_t toffset = TSDB_KEY_COL_OFFSET;
     int32_t  tlen = pBlock->keyLen;
 
     if (dcol != 0) {
       SBlockCol *pBlockCol = &(pBlockData->cols[ccol]);
       tcolId = pBlockCol->colId;
-      toffset = pBlockCol->offset;
+      toffset = tsdbGetBlockColOffset(pBlockCol);
       tlen = pBlockCol->len;
     } else {
       ASSERT(pDataCol->colId == tcolId);
@@ -488,7 +488,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
                                        pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
                                        (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
-        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d",
+        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u",
                   TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
         return -1;
       }
@@ -627,7 +627,7 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
   if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
   if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
 
-  int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + pBlockCol->offset;
+  int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + tsdbGetBlockColOffset(pBlockCol);
   if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
     tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s",
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));