Skip to content

TDengine
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
TDengine
TDengine
提交 f4aaf791 编写于 作者: C Cary Xu
浏览文件
操作

fix: check commit and merge and add cases

上级 56df9f6c
隐藏空白更改
内联 并排
Showing with 322 addition and 19 deletion
include/common/taosdef.h
浏览文件 @ f4aaf791
......@@ -86,6 +86,11 @@ typedef enum {
TSDB_RETENTION_MAX = 3
} ERetentionLevel;
typedef enum {
TSDB_BITMODE_DEFAULT = 0, // 2 bits
TSDB_BITMODE_ONE_BIT = 1, // 1 bit
} EBitmapMode;
extern char *qtypeStr[];
#define TSDB_PORT_HTTP 11
......
include/common/tdataformat.h
浏览文件 @ f4aaf791
......@@ -313,8 +313,9 @@ typedef struct {
SDataCol *cols;
} SDataCols;
static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != 0; }
static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = 1; }
static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != TSDB_BITMODE_DEFAULT; }
static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = TSDB_BITMODE_ONE_BIT; }
static FORCE_INLINE bool tdIsBitmapModeI(int8_t bitmapMode) { return bitmapMode != TSDB_BITMODE_DEFAULT; }
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column
#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data
......
source/common/src/tdataformat.c
浏览文件 @ f4aaf791
......@@ -855,7 +855,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
pCols->maxCols = maxCols;
pCols->numOfRows = 0;
pCols->numOfCols = 0;
// pCols->bitmapMode = 0; // calloc already set 0
pCols->bitmapMode = TSDB_BITMODE_DEFAULT;
if (maxCols > 0) {
pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol));
......@@ -899,7 +899,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
#endif
pCols->numOfRows = 0;
pCols->bitmapMode = 0;
pCols->bitmapMode = TSDB_BITMODE_DEFAULT;
pCols->numOfCols = schemaNCols(pSchema);
for (i = 0; i < schemaNCols(pSchema); ++i) {
......
source/common/src/trow.c
浏览文件 @ f4aaf791
......@@ -341,18 +341,19 @@ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8
bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) {
int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I);
uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM];
uint8_t *qBitmap = (uint8_t*)pBitmap;
for (int i = 0; i < nBytes; ++i) {
if (*((uint8_t *)pBitmap) != vTypeByte) {
if (*qBitmap != vTypeByte) {
return false;
}
pBitmap = POINTER_SHIFT(pBitmap, i);
qBitmap = (uint8_t *)POINTER_SHIFT(pBitmap, i);
}
int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
for (int j = 0; j < nLeft; ++j) {
uint8_t vType;
tdGetBitmapValType(pBitmap, j, &vType, bitmapMode);
tdGetBitmapValType(qBitmap, j, &vType, bitmapMode);
if (vType != TD_VTYPE_NORM) {
return false;
}
......
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ f4aaf791
......@@ -1138,6 +1138,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
memcpy(tptr, pDataCol->pData, flen);
if (tBitmaps > 0) {
bptr = POINTER_SHIFT(pBlockData, lsize + flen);
if (isSuper && !tdDataColsIsBitmapI(pDataCols)) {
tdMergeBitmap((uint8_t *)pDataCol->pBitmap, rowsToWrite, (uint8_t *)pDataCol->pBitmap);
}
memcpy(bptr, pDataCol->pBitmap, tBitmaps);
tBitmapsLen = tBitmaps;
flen += tBitmapsLen;
......@@ -1503,13 +1506,16 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
tSkipListIterNext(pCommitIter->pIter);
} else {
if (lastKey != key1) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pTarget->numOfRows;
}
lastKey = key1;
++pTarget->numOfRows;
}
// copy disk data
for (int i = 0; i < pDataCols->numOfCols; ++i) {
SCellVal sVal = {0};
// no duplicated TS keys in pDataCols from file
if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
TASSERT(0);
}
......
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ f4aaf791
......@@ -1965,7 +1965,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID &&
cur->pos >= 0 && cur->pos < pBlock->numOfRows);
// Even Multi-Version supported, the records with duplicated TSKEY would be merged inside of tsdbLoadData interface.
TSKEY* tsArray = pCols->cols[0].pData;
assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst &&
tsArray[pBlock->numOfRows - 1] == pBlock->keyLast);
......@@ -1995,6 +1995,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
int32_t pos = cur->pos;
cur->win = TSWINDOW_INITIALIZER;
bool adjustPos = false;
// no data in buffer, load data from file directly
if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
......@@ -2016,6 +2017,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
break;
}
if (adjustPos) {
if (key == lastKeyAppend) {
pos -= step;
}
adjustPos = false;
}
if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) ||
((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) {
break;
......@@ -2107,7 +2115,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
moveToNextRowInMem(pCheckInfo);
pos += step;
adjustPos = true;
} else {
// discard the memory record
moveToNextRowInMem(pCheckInfo);
}
} else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) {
......
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
浏览文件 @ f4aaf791
......@@ -20,11 +20,11 @@
static void tsdbResetReadTable(SReadH *pReadh);
static void tsdbResetReadFile(SReadH *pReadh);
static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock);
static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols);
static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode);
static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp,
int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize);
static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
int numOfColIds);
int numOfColIds, int8_t bitmapMode);
static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) {
......@@ -266,10 +266,11 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
}
}
if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1;
if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0], TSDB_BITMODE_ONE_BIT) < 0) return -1;
for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
iBlock++;
if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1], TSDB_BITMODE_DEFAULT) < 0) return -1;
// TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
......@@ -309,10 +310,10 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
}
}
if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1;
if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds, TSDB_BITMODE_ONE_BIT) < 0) return -1;
for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
iBlock++;
if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds, TSDB_BITMODE_DEFAULT) < 0) return -1;
// TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
......@@ -543,14 +544,14 @@ static void tsdbResetReadFile(SReadH *pReadh) {
tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
}
static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) {
static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode) {
ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
tdResetDataCols(pDataCols);
if (tsdbIsSupBlock(pBlock)) {
if (tdIsBitmapModeI(bitmapMode)) {
tdDataColsSetBitmapI(pDataCols);
}
......@@ -730,7 +731,7 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
}
static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
int numOfColIds) {
int numOfColIds, int8_t bitmapMode) {
ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID);
......@@ -739,7 +740,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
tdResetDataCols(pDataCols);
if (tsdbIsSupBlock(pBlock)) {
if (tdIsBitmapModeI(bitmapMode)) {
tdDataColsSetBitmapI(pDataCols);
}
......
tests/script/tsim/insert/commit-merge0.sim 0 → 100644
浏览文件 @ f4aaf791
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print =============== create database
sql create database db days 300 keep 365000d,365000d,365000d
sql show databases
if $rows != 3 then
return -1
endi
print $data00 $data01 $data02
sql use db
sql create table stb1(ts timestamp, c6 double) tags (t1 int);
sql create table ct1 using stb1 tags ( 1 );
sql create table ct2 using stb1 tags ( 2 );
sql create table ct3 using stb1 tags ( 3 );
sql create table ct4 using stb1 tags ( 4 );
sql insert into ct1 values ('2022-05-01 18:30:27.001', 0.0);
sql insert into ct4 values ('2022-04-28 18:30:27.002', 0.0);
sql insert into ct1 values ('2022-05-01 18:30:17.003', 11.11);
sql insert into ct4 values ('2022-02-01 18:30:27.004', 11.11);
sql insert into ct1 values ('2022-05-01 18:30:07.005', 22.22);
sql insert into ct4 values ('2021-11-01 18:30:27.006', 22.22);
sql insert into ct1 values ('2022-05-01 18:29:27.007', 33.33);
sql insert into ct4 values ('2022-08-01 18:30:27.008', 33.33);
sql insert into ct1 values ('2022-05-01 18:20:27.009', 44.44);
sql insert into ct4 values ('2021-05-01 18:30:27.010', 44.44);
sql insert into ct1 values ('2022-05-01 18:21:27.011', 55.55);
sql insert into ct4 values ('2021-01-01 18:30:27.012', 55.55);
sql insert into ct1 values ('2022-05-01 18:22:27.013', 66.66);
sql insert into ct4 values ('2020-06-01 18:30:27.014', 66.66);
sql insert into ct1 values ('2022-05-01 18:28:37.015', 77.77);
sql insert into ct4 values ('2020-05-01 18:30:27.016', 77.77);
sql insert into ct1 values ('2022-05-01 18:29:17.017', 88.88);
sql insert into ct4 values ('2019-05-01 18:30:27.018', 88.88);
sql insert into ct1 values ('2022-05-01 18:30:20.019', 0);
sql insert into ct1 values ('2022-05-01 18:30:47.020', -99.99);
sql insert into ct1 values ('2022-05-01 18:30:49.021', NULL);
sql insert into ct1 values ('2022-05-01 18:30:51.022', -99.99);
sql insert into ct4 values ('2018-05-01 18:30:27.023', NULL) ;
sql insert into ct4 values ('2021-03-01 18:30:27.024', NULL) ;
sql insert into ct4 values ('2022-08-01 18:30:27.025', NULL) ;
print =============== select * from ct1 - memory
sql select * from stb1;
if $rows != 25 then
print rows = $rows != 25
return -1
endi
print =============== stop and restart taosd
$reboot_max = 10;
$reboot_cnt = 0
reboot_and_check:
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
print =============== insert duplicated records to memory - loop $reboot_max - $reboot_cnt
sql use db
sql insert into ct1 values ('2022-05-01 18:30:27.001', 0.0);
sql insert into ct4 values ('2022-04-28 18:30:27.002', 0.0);
sql insert into ct1 values ('2022-05-01 18:30:17.003', 11.11);
sql insert into ct4 values ('2022-02-01 18:30:27.004', 11.11);
sql insert into ct1 values ('2022-05-01 18:30:07.005', 22.22);
sql insert into ct4 values ('2021-11-01 18:30:27.006', 22.22);
sql insert into ct1 values ('2022-05-01 18:29:27.007', 33.33);
sql insert into ct4 values ('2022-08-01 18:30:27.008', 33.33);
sql insert into ct1 values ('2022-05-01 18:20:27.009', 44.44);
sql insert into ct4 values ('2021-05-01 18:30:27.010', 44.44);
sql insert into ct1 values ('2022-05-01 18:21:27.011', 55.55);
sql insert into ct4 values ('2021-01-01 18:30:27.012', 55.55);
sql insert into ct1 values ('2022-05-01 18:22:27.013', 66.66);
sql insert into ct4 values ('2020-06-01 18:30:27.014', 66.66);
sql insert into ct1 values ('2022-05-01 18:28:37.015', 77.77);
sql insert into ct4 values ('2020-05-01 18:30:27.016', 77.77);
sql insert into ct1 values ('2022-05-01 18:29:17.017', 88.88);
sql insert into ct4 values ('2019-05-01 18:30:27.018', 88.88);
sql insert into ct1 values ('2022-05-01 18:30:20.019', 0);
sql insert into ct1 values ('2022-05-01 18:30:47.020', -99.99);
sql insert into ct1 values ('2022-05-01 18:30:49.021', NULL);
sql insert into ct1 values ('2022-05-01 18:30:51.022', -99.99);
sql insert into ct4 values ('2018-05-01 18:30:27.023', NULL) ;
sql insert into ct4 values ('2021-03-01 18:30:27.024', NULL) ;
sql insert into ct4 values ('2022-08-01 18:30:27.025', NULL) ;
print =============== select * from ct1 - merge memory and file - loop $reboot_max - $reboot_cnt
sql select * from ct1;
if $rows != 13 then
print rows = $rows != 13
return -1
endi
print $data[0][0] $data[0][1]
print $data[1][0] $data[1][1]
print $data[2][0] $data[2][1]
print $data[3][0] $data[3][1]
print $data[4][0] $data[4][1]
print $data[5][0] $data[5][1]
print $data[6][0] $data[6][1]
print $data[7][0] $data[7][1]
print $data[8][0] $data[8][1]
print $data[9][0] $data[9][1]
print $data[10][0] $data[10][1]
print $data[11][0] $data[11][1]
print $data[12][0] $data[12][1]
if $data[0][1] != 44.440000000 then
print $data[0][1] != 44.440000000
return -1
endi
if $data[1][1] != 55.550000000 then
print $data[1][1] != 55.550000000
return -1
endi
if $data[2][1] != 66.660000000 then
print $data[2][1] != 66.660000000
return -1
endi
if $data[3][1] != 77.770000000 then
print $data[3][1] != 77.770000000
return -1
endi
if $data[4][1] != 88.880000000 then
print $data[4][1] != 88.880000000
return -1
endi
if $data[5][1] != 33.330000000 then
print $data[5][1] != 33.330000000
return -1
endi
if $data[6][1] != 22.220000000 then
print $data[6][1] != 22.220000000
return -1
endi
if $data[7][1] != 11.110000000 then
print $data[7][1] != 11.110000000
return -1
endi
if $data[8][1] != 0.000000000 then
print $data[8][1] != 0.000000000
return -1
endi
if $data[9][1] != 0.000000000 then
print $data[9][1] != 0.000000000
return -1
endi
if $data[10][1] != -99.990000000 then
print $data[10][1] != -99.990000000
return -1
endi
if $data[11][1] != NULL then
print $data[11][1] != NULL
return -1
endi
if $data[12][1] != -99.990000000 then
print $data[12][1] != -99.990000000
return -1
endi
print =============== select * from ct4 - merge memory and file - loop $reboot_max - $reboot_cnt
sql select * from ct4;
if $rows != 12 then
print rows = $rows != 12
return -1
endi
print $data[0][0] $data[0][1]
print $data[1][0] $data[1][1]
print $data[2][0] $data[2][1]
print $data[3][0] $data[3][1]
print $data[4][0] $data[4][1]
print $data[5][0] $data[5][1]
print $data[6][0] $data[6][1]
print $data[7][0] $data[7][1]
print $data[8][0] $data[8][1]
print $data[9][0] $data[9][1]
print $data[10][0] $data[10][1]
print $data[11][0] $data[11][1]
if $data[0][1] != NULL then
print $data[0][1] != NULL
return -1
endi
if $data[1][1] != 88.880000000 then
print $data[1][1] != 88.880000000
return -1
endi
if $data[2][1] != 77.770000000 then
print $data[2][1] != 77.770000000
return -1
endi
if $data[3][1] != 66.660000000 then
print $data[3][1] != 66.660000000
return -1
endi
if $data[4][1] != 55.550000000 then
print $data[4][1] != 55.550000000
return -1
endi
if $data[5][1] != NULL then
print $data[5][1] != NULL
return -1
endi
if $data[6][1] != 44.440000000 then
print $data[6][1] != 44.440000000
return -1
endi
if $data[7][1] != 22.220000000 then
print $data[7][1] != 22.220000000
return -1
endi
if $data[8][1] != 11.110000000 then
print $data[8][1] != 11.110000000
return -1
endi
if $data[9][1] != 0.000000000 then
print $data[9][1] != 0.000000000
return -1
endi
if $data[10][1] != 33.330000000 then
print $data[10][1] != 33.330000000
return -1
endi
if $data[11][1] != NULL then
print $data[11][1] != NULL
return -1
endi
if $reboot_cnt > $reboot_max then
print reboot_cnt $reboot_cnt > reboot_max $reboot_max
return 0
else
print reboot_cnt $reboot_cnt <= reboot_max $reboot_max
$reboot_cnt = $reboot_cnt + 1
goto reboot_and_check
endi
tests/script/tsim/insert/update0.sim
浏览文件 @ f4aaf791
......@@ -99,6 +99,23 @@ endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
print =============== step3-2 query records of ct1 from file
sql select * from ct1;
print $data00 $data01
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或