From 5978d7081feb3764e61b59dc35060a71773b9848 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 25 Feb 2021 13:55:45 +0800
Subject: [PATCH] [TD-3075]<hotfix>: fix SBlockCol offset overflow

---
 src/tsdb/inc/tsdbReadImpl.h | 35 ++++++++++++++++++++++++-----------
 src/tsdb/src/tsdbCommit.c   | 12 ++++++------
 src/tsdb/src/tsdbReadImpl.c | 12 ++++++------
 3 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/src/tsdb/inc/tsdbReadImpl.h b/src/tsdb/inc/tsdbReadImpl.h
index 0efbcc55bb..e7840d9723 100644
--- a/src/tsdb/inc/tsdbReadImpl.h
+++ b/src/tsdb/inc/tsdbReadImpl.h
@@ -49,19 +49,32 @@ typedef struct {
 } SBlockInfo;
 
 typedef struct {
-  int16_t colId;
-  int32_t len;
-  int32_t type : 8;
-  int32_t offset : 24;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-  char    padding[2];
+  int16_t  colId;
+  int32_t  len;
+  uint32_t type : 8;
+  uint32_t offset : 24;
+  int64_t  sum;
+  int64_t  max;
+  int64_t  min;
+  int16_t  maxIndex;
+  int16_t  minIndex;
+  int16_t  numOfNull;
+  uint8_t  offsetH;
+  char     padding[1];
 } SBlockCol;
 
+// Code here just for back-ward compatibility
+static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
+  pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
+  pBlockCol->offsetH = (uint8_t)(offset >> 24);
+}
+
+static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
+  uint32_t offset1 = pBlockCol->offset;
+  uint32_t offset2 = pBlockCol->offsetH;
+  return (offset1 | (offset2 << 24));
+}
+
 typedef struct {
   int32_t   delimiter;  // For recovery usage
   int32_t   numOfCols;  // For recovery usage
diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c
index a777b11186..a4cc316725 100644
--- a/src/tsdb/src/tsdbCommit.c
+++ b/src/tsdb/src/tsdbCommit.c
@@ -781,11 +781,11 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo
   ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols);
 
   // Compress the data if neccessary
-  int     tcol = 0;  // counter of not all NULL and written columns
-  int32_t toffset = 0;
-  int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull);
-  int32_t lsize = tsize;
-  int32_t keyLen = 0;
+  int      tcol = 0;  // counter of not all NULL and written columns
+  uint32_t toffset = 0;
+  int32_t  tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull);
+  int32_t  lsize = tsize;
+  int32_t  keyLen = 0;
   for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
     // All not NULL columns finish
     if (ncol != 0 && tcol >= nColsNotAllNull) break;
@@ -829,7 +829,7 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo
     tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)));
 
     if (ncol != 0) {
-      pBlockCol->offset = toffset;
+      tsdbSetBlockColOffset(pBlockCol, toffset);
       pBlockCol->len = flen;
       tcol++;
     } else {
diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c
index 312f1f9b20..572706d45e 100644
--- a/src/tsdb/src/tsdbReadImpl.c
+++ b/src/tsdb/src/tsdbReadImpl.c
@@ -466,14 +466,14 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       continue;
     }
 
-    int16_t tcolId = 0;
-    int32_t toffset = TSDB_KEY_COL_OFFSET;
-    int32_t tlen = pBlock->keyLen;
+    int16_t  tcolId = 0;
+    uint32_t toffset = TSDB_KEY_COL_OFFSET;
+    int32_t  tlen = pBlock->keyLen;
 
     if (dcol != 0) {
       SBlockCol *pBlockCol = &(pBlockData->cols[ccol]);
       tcolId = pBlockCol->colId;
-      toffset = pBlockCol->offset;
+      toffset = tsdbGetBlockColOffset(pBlockCol);
       tlen = pBlockCol->len;
     } else {
       ASSERT(pDataCol->colId == tcolId);
@@ -488,7 +488,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
                                        pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
                                        (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
-        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d",
+        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u",
                   TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
         return -1;
       }
@@ -627,7 +627,7 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
   if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
   if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
 
-  int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + pBlockCol->offset;
+  int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + tsdbGetBlockColOffset(pBlockCol);
   if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
     tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s",
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));
-- 
GitLab