Skip to content

T
TDengine

taosdata / TDengine
接近 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 e12576ed 编写于 作者: S shenglian zhou
浏览文件
操作

Merge branch '3.0' of github.com:taosdata/TDengine into szhou/feature/multiwaymerge

上级 d5a29fc4 4935b146
展开全部 隐藏空白更改
内联 并排
Showing with 1172 addition and 1773 deletion
include/libs/function/functionMgt.h
浏览文件 @ e12576ed
...@@ -122,6 +122,12 @@ typedef enum EFunctionType { ...@@ -122,6 +122,12 @@ typedef enum EFunctionType {
// internal function // internal function
FUNCTION_TYPE_SELECT_VALUE, FUNCTION_TYPE_SELECT_VALUE,
// distributed splitting functions
FUNCTION_TYPE_APERCENTILE_PARTIAL,
FUNCTION_TYPE_APERCENTILE_MERGE,
FUNCTION_TYPE_SPREAD_PARTIAL,
FUNCTION_TYPE_SPREAD_MERGE,
// user defined funcion // user defined funcion
FUNCTION_TYPE_UDF = 10000 FUNCTION_TYPE_UDF = 10000
} EFunctionType; } EFunctionType;
......
include/util/talgo.h
浏览文件 @ e12576ed
...@@ -65,7 +65,7 @@ void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ ...@@ -65,7 +65,7 @@ void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __
* @param flags * @param flags
* @return * @return
*/ */
void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t fn, int32_t flags); void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size, __compar_fn_t compar, int32_t flags);
/** /**
* adjust heap * adjust heap
...@@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size ...@@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
* @return * @return
*/ */
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char* buf, bool maxroot); __ext_compar_fn_t compar, char *buf, bool maxroot);
/** /**
* sort heap to make sure it is a max/min root heap * sort heap to make sure it is a max/min root heap
...@@ -97,8 +97,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const ...@@ -97,8 +97,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
* @param maxroot: if heap is max root heap * @param maxroot: if heap is max root heap
* @return * @return
*/ */
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot);
bool maxroot);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
include/util/tdef.h
浏览文件 @ e12576ed
...@@ -32,7 +32,7 @@ extern "C" { ...@@ -32,7 +32,7 @@ extern "C" {
#define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle #define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle
// Bytes for each type. // Bytes for each type.
extern const int32_t TYPE_BYTES[15]; extern const int32_t TYPE_BYTES[16];
// TODO: replace and remove code below // TODO: replace and remove code below
#define CHAR_BYTES sizeof(char) #define CHAR_BYTES sizeof(char)
......
source/client/src/clientImpl.c
浏览文件 @ e12576ed
...@@ -1280,8 +1280,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int ...@@ -1280,8 +1280,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
varDataVal(dst) + CHAR_BYTES); varDataVal(dst) + CHAR_BYTES);
if (length <= 0) { if (length <= 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
varDataVal(jsonInnerData));
length = 0; length = 0;
} }
varDataSetLen(dst, length + CHAR_BYTES * 2); varDataSetLen(dst, length + CHAR_BYTES * 2);
......
source/common/src/tdatablock.c
浏览文件 @ e12576ed
...@@ -1131,6 +1131,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui ...@@ -1131,6 +1131,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui
if (IS_VAR_DATA_TYPE(pColumn->info.type)) { if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
...@@ -1156,6 +1157,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui ...@@ -1156,6 +1157,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
memset(tmp + pColumn->info.bytes * existRows, 0, pColumn->info.bytes * (numOfRows - existRows));
pColumn->pData = tmp; pColumn->pData = tmp;
} }
...@@ -1269,12 +1271,12 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { ...@@ -1269,12 +1271,12 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
void colDataDestroy(SColumnInfoData* pColData) { void colDataDestroy(SColumnInfoData* pColData) {
if (IS_VAR_DATA_TYPE(pColData->info.type)) { if (IS_VAR_DATA_TYPE(pColData->info.type)) {
taosMemoryFree(pColData->varmeta.offset); taosMemoryFreeClear(pColData->varmeta.offset);
} else { } else {
taosMemoryFree(pColData->nullbitmap); taosMemoryFreeClear(pColData->nullbitmap);
} }
taosMemoryFree(pColData->pData); taosMemoryFreeClear(pColData->pData);
} }
static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
......
source/common/src/tglobal.c
浏览文件 @ e12576ed
...@@ -332,7 +332,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { ...@@ -332,7 +332,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
if (cfgAddString(pCfg, "smlTagNullName", tsSmlTagName, 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 4; tsNumOfTaskQueueThreads = tsNumOfCores / 4;
...@@ -532,7 +532,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { ...@@ -532,7 +532,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
} }
tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagNullName")->str, TSDB_COL_NAME_LEN); tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
......
source/common/src/ttypes.c
浏览文件 @ e12576ed
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
#include "tcompression.h" #include "tcompression.h"
#include "trow.h" #include "trow.h"
const int32_t TYPE_BYTES[15] = { const int32_t TYPE_BYTES[16] = {
-1, // TSDB_DATA_TYPE_NULL -1, // TSDB_DATA_TYPE_NULL
CHAR_BYTES, // TSDB_DATA_TYPE_BOOL CHAR_BYTES, // TSDB_DATA_TYPE_BOOL
CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT
...@@ -34,6 +34,7 @@ const int32_t TYPE_BYTES[15] = { ...@@ -34,6 +34,7 @@ const int32_t TYPE_BYTES[15] = {
SHORT_BYTES, // TSDB_DATA_TYPE_USMALLINT SHORT_BYTES, // TSDB_DATA_TYPE_USMALLINT
INT_BYTES, // TSDB_DATA_TYPE_UINT INT_BYTES, // TSDB_DATA_TYPE_UINT
sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT
TSDB_MAX_JSON_TAG_LEN, // TSDB_DATA_TYPE_JSON
}; };
#define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \ #define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \
......
source/dnode/vnode/CMakeLists.txt
浏览文件 @ e12576ed
...@@ -36,12 +36,10 @@ target_sources( ...@@ -36,12 +36,10 @@ target_sources(
# tsdb # tsdb
"src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit.c"
# "src/tsdb/tsdbCommit2.c"
"src/tsdb/tsdbFile.c" "src/tsdb/tsdbFile.c"
"src/tsdb/tsdbFS.c" "src/tsdb/tsdbFS.c"
"src/tsdb/tsdbOpen.c" "src/tsdb/tsdbOpen.c"
"src/tsdb/tsdbMemTable.c" "src/tsdb/tsdbMemTable.c"
# "src/tsdb/tsdbMemTable2.c"
"src/tsdb/tsdbRead.c" "src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbReadImpl.c"
"src/tsdb/tsdbWrite.c" "src/tsdb/tsdbWrite.c"
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ e12576ed
...@@ -58,28 +58,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, ST ...@@ -58,28 +58,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, ST
bool tsdbTbDataIterNext(STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter);
bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow); bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow);
int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
// tsdbMemTable2.c ==============================================================================================
// typedef struct SMemTable2 SMemTable2;
// typedef struct SMemData SMemData;
// typedef struct SMemDataIter SMemDataIter;
// int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable);
// void tsdbMemTableDestroy2(SMemTable2 *pMemTable);
// int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk);
// int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
// /* SMemDataIter */
// void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter);
// bool tsdbMemDataIterNext(SMemDataIter *pIter);
// void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow);
// // tsdbCommit2.c ==============================================================================================
// int32_t tsdbBegin2(STsdb *pTsdb);
// int32_t tsdbCommit2(STsdb *pTsdb);
// tsdbFile.c ============================================================================================== // tsdbFile.c ==============================================================================================
typedef int32_t TSDB_FILE_T; typedef int32_t TSDB_FILE_T;
typedef struct SDFInfo SDFInfo; typedef struct SDFInfo SDFInfo;
...@@ -700,17 +678,6 @@ typedef struct { ...@@ -700,17 +678,6 @@ typedef struct {
TSKEY eKey; TSKEY eKey;
} SDelInfo; } SDelInfo;
struct SMemTable2 {
STsdb *pTsdb;
int32_t nRef;
TSDBKEY minKey;
TSDBKEY maxKey;
int64_t nRows;
int64_t nDelOp;
SArray *aSkmInfo;
SArray *aMemData;
};
static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
TSDBKEY *pKey1 = (TSDBKEY *)p1; TSDBKEY *pKey1 = (TSDBKEY *)p1;
TSDBKEY *pKey2 = (TSDBKEY *)p2; TSDBKEY *pKey2 = (TSDBKEY *)p2;
...@@ -730,24 +697,6 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { ...@@ -730,24 +697,6 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
return 0; return 0;
} }
struct SMemData {
tb_uid_t suid;
tb_uid_t uid;
TSDBKEY minKey;
TSDBKEY maxKey;
SDelOp *delOpHead;
SDelOp *delOpTail;
SMemSkipList sl;
};
struct SMemDataIter {
STbData *pMemData;
int8_t backward;
TSDBROW *pRow;
SMemSkipListNode *pNode; // current node
TSDBROW row;
};
struct STbDataIter { struct STbDataIter {
STbData *pTbData; STbData *pTbData;
int8_t backward; int8_t backward;
......
source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @ e12576ed
...@@ -112,7 +112,7 @@ int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); ...@@ -112,7 +112,7 @@ int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
// tsdb // tsdb
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg);
int tsdbClose(STsdb** pTsdb); int tsdbClose(STsdb** pTsdb);
int tsdbBegin(STsdb* pTsdb); int32_t tsdbBegin(STsdb* pTsdb);
int32_t tsdbCommit(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
...@@ -161,18 +161,6 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore); ...@@ -161,18 +161,6 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
void tdUidStoreDestory(STbUidStore* pStore); void tdUidStoreDestory(STbUidStore* pStore);
void* tdUidStoreFree(STbUidStore* pStore); void* tdUidStoreFree(STbUidStore* pStore);
#if 0
int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
void tsdbUidStoreDestory(STbUidStore* pStore);
void* tsdbUidStoreFree(STbUidStore* pStore);
int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType);
#endif
typedef struct { typedef struct {
int8_t streamType; // sma or other int8_t streamType; // sma or other
int8_t dstType; int8_t dstType;
......
source/dnode/vnode/src/meta/metaTable.c
浏览文件 @ e12576ed
...@@ -61,14 +61,14 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const ...@@ -61,14 +61,14 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
if (tTagToValArray((const STag *)data, &pTagVals) != 0) { if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
return -1; return -1;
} }
char key[512] = {0};
SIndexMultiTerm *terms = indexMultiTermCreate(); SIndexMultiTerm *terms = indexMultiTermCreate();
int16_t nCols = taosArrayGetSize(pTagVals); int16_t nCols = taosArrayGetSize(pTagVals);
for (int i = 0; i < nCols; i++) { for (int i = 0; i < nCols; i++) {
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type; char type = pTagVal->type;
sprintf(key, "%s_%s", tagName, pTagVal->pKey);
char * key = pTagVal->pKey;
int32_t nKey = strlen(key); int32_t nKey = strlen(key);
SIndexTerm *term = NULL; SIndexTerm *term = NULL;
...@@ -93,12 +93,11 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const ...@@ -93,12 +93,11 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
} else if (type == TSDB_DATA_TYPE_BOOL) { } else if (type == TSDB_DATA_TYPE_BOOL) {
int val = *(int *)(&pTagVal->i64); int val = *(int *)(&pTagVal->i64);
int len = 0; int len = 0;
term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len);
} }
if (term != NULL) { if (term != NULL) {
indexMultiTermAdd(terms, term); indexMultiTermAdd(terms, term);
} }
memset(key, 0, sizeof(key));
} }
tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid); tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
indexMultiTermDestroy(terms); indexMultiTermDestroy(terms);
......
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ e12576ed
...@@ -28,6 +28,8 @@ typedef struct { ...@@ -28,6 +28,8 @@ typedef struct {
int niters; // memory iterators int niters; // memory iterators
SCommitIter *iters; SCommitIter *iters;
bool isRFileSet; // read and commit FSET bool isRFileSet; // read and commit FSET
int32_t fid;
SDFileSet *pSet;
SReadH readh; SReadH readh;
SDFileSet wSet; SDFileSet wSet;
bool isDFileSame; bool isDFileSame;
...@@ -58,8 +60,12 @@ typedef struct { ...@@ -58,8 +60,12 @@ typedef struct {
#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows) #define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows)
#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
static void tsdbStartCommit(STsdb *pRepo); static int32_t tsdbCommitData(SCommitH *pCommith);
static void tsdbEndCommit(STsdb *pTsdb, int eno); static int32_t tsdbCommitDel(SCommitH *pCommith);
static int32_t tsdbCommitCache(SCommitH *pCommith);
static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitH *pCHandle);
static int32_t tsdbEndCommit(SCommitH *pCHandle, int eno);
static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo); static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo);
static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key);
static int tsdbNextCommitFid(SCommitH *pCommith); static int tsdbNextCommitFid(SCommitH *pCommith);
...@@ -67,7 +73,6 @@ static void tsdbDestroyCommitH(SCommitH *pCommith); ...@@ -67,7 +73,6 @@ static void tsdbDestroyCommitH(SCommitH *pCommith);
static int32_t tsdbCreateCommitIters(SCommitH *pCommith); static int32_t tsdbCreateCommitIters(SCommitH *pCommith);
static void tsdbDestroyCommitIters(SCommitH *pCommith); static void tsdbDestroyCommitIters(SCommitH *pCommith);
static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static void tsdbResetCommitFile(SCommitH *pCommith);
static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static int tsdbCommitToTable(SCommitH *pCommith, int tid); static int tsdbCommitToTable(SCommitH *pCommith, int tid);
static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx); static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx);
...@@ -88,8 +93,11 @@ static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *i ...@@ -88,8 +93,11 @@ static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *i
SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update); SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update);
static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn); static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn);
static int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup,
SMergeInfo *pMergeInfo);
int tsdbBegin(STsdb *pTsdb) { int32_t tsdbBegin(STsdb *pTsdb) {
if (!pTsdb) return 0; if (!pTsdb) return 0;
SMemTable *pMem; SMemTable *pMem;
...@@ -112,15 +120,50 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -112,15 +120,50 @@ int32_t tsdbCommit(STsdb *pTsdb) {
pTsdb->mem = NULL; pTsdb->mem = NULL;
// start commit // start commit
tsdbStartCommit(pTsdb); code = tsdbStartCommit(pTsdb, &commith);
if (tsdbInitCommitH(&commith, pTsdb) < 0) { if (code) {
return -1; goto _err;
}
// commit impl
code = tsdbCommitData(&commith);
if (code) {
goto _err;
} }
code = tsdbCommitDel(&commith);
if (code) {
goto _err;
}
code = tsdbCommitCache(&commith);
if (code) {
goto _err;
}
// end commit
code = tsdbEndCommit(&commith, 0);
if (code) {
goto _err;
}
return code;
_err:
tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
static int32_t tsdbCommitData(SCommitH *pCommith) {
int32_t fid;
SDFileSet *pSet = NULL;
int32_t code = 0;
STsdb *pTsdb = TSDB_COMMIT_REPO(pCommith);
// Skip expired memory data and expired FSET // Skip expired memory data and expired FSET
tsdbSeekCommitIter(&commith, commith.rtn.minKey); tsdbSeekCommitIter(pCommith, pCommith->rtn.minKey);
while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { while ((pSet = tsdbFSIterNext(&(pCommith->fsIter)))) {
if (pSet->fid < commith.rtn.minFid) { if (pSet->fid < pCommith->rtn.minFid) {
tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pTsdb), pSet->fid, tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pTsdb), pSet->fid,
TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
} else { } else {
...@@ -129,7 +172,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -129,7 +172,7 @@ int32_t tsdbCommit(STsdb *pTsdb) {
} }
// commit // commit
fid = tsdbNextCommitFid(&(commith)); fid = tsdbNextCommitFid(pCommith);
while (true) { while (true) {
// Loop over both on disk and memory // Loop over both on disk and memory
if (pSet == NULL && fid == TSDB_IVLD_FID) break; if (pSet == NULL && fid == TSDB_IVLD_FID) break;
...@@ -137,12 +180,12 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -137,12 +180,12 @@ int32_t tsdbCommit(STsdb *pTsdb) {
if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) {
// Only has existing FSET but no memory data to commit in this // Only has existing FSET but no memory data to commit in this
// existing FSET, only check if file in correct retention // existing FSET, only check if file in correct retention
if (tsdbApplyRtnOnFSet(pTsdb, pSet, &(commith.rtn)) < 0) { if (tsdbApplyRtnOnFSet(TSDB_COMMIT_REPO(pCommith), pSet, &(pCommith->rtn)) < 0) {
tsdbDestroyCommitH(&commith); tsdbDestroyCommitH(pCommith);
return -1; return -1;
} }
pSet = tsdbFSIterNext(&(commith.fsIter)); pSet = tsdbFSIterNext(&(pCommith->fsIter));
} else { } else {
// Has memory data to commit // Has memory data to commit
SDFileSet *pCSet; SDFileSet *pCSet;
...@@ -156,22 +199,30 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -156,22 +199,30 @@ int32_t tsdbCommit(STsdb *pTsdb) {
// Commit to an existing FSET // Commit to an existing FSET
pCSet = pSet; pCSet = pSet;
cfid = pSet->fid; cfid = pSet->fid;
pSet = tsdbFSIterNext(&(commith.fsIter)); pSet = tsdbFSIterNext(&(pCommith->fsIter));
} }
if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { if (tsdbCommitToFile(pCommith, pCSet, cfid) < 0) {
tsdbDestroyCommitH(&commith); tsdbDestroyCommitH(pCommith);
return -1; return -1;
} }
fid = tsdbNextCommitFid(&commith); fid = tsdbNextCommitFid(pCommith);
} }
} }
// end commit return code;
tsdbDestroyCommitH(&commith); }
tsdbEndCommit(pTsdb, TSDB_CODE_SUCCESS);
static int32_t tsdbCommitDel(SCommitH *pCommith) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitCache(SCommitH *pCommith) {
int32_t code = 0;
// TODO
return code; return code;
} }
...@@ -216,16 +267,6 @@ static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { ...@@ -216,16 +267,6 @@ static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
return 0; return 0;
} }
// int tsdbPrepareCommit(STsdb *pTsdb) {
// if (pTsdb->mem == NULL) return 0;
// ASSERT(pTsdb->imem == NULL);
// pTsdb->imem = pTsdb->mem;
// pTsdb->mem = NULL;
// return 0;
// }
void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
TSKEY minKey, midKey, maxKey, now; TSKEY minKey, midKey, maxKey, now;
...@@ -243,19 +284,32 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { ...@@ -243,19 +284,32 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
pRtn->minFid, pRtn->midFid, pRtn->maxFid); pRtn->minFid, pRtn->midFid, pRtn->maxFid);
} }
static void tsdbStartCommit(STsdb *pRepo) { static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitH *pCHandle) {
SMemTable *pMem = pRepo->imem; int32_t code = 0;
tsdbInfo("vgId:%d, start to commit", REPO_ID(pTsdb));
if (tsdbInitCommitH(pCHandle, pTsdb) < 0) {
return -1;
}
tsdbInfo("vgId:%d, start to commit", REPO_ID(pRepo)); tsdbStartFSTxn(pTsdb, 0, 0);
tsdbStartFSTxn(pRepo, 0, 0); return code;
} }
static void tsdbEndCommit(STsdb *pTsdb, int eno) { static int32_t tsdbEndCommit(SCommitH *pCHandle, int eno) {
int32_t code = 0;
STsdb *pTsdb = TSDB_COMMIT_REPO(pCHandle);
tsdbDestroyCommitH(pCHandle);
tsdbEndFSTxn(pTsdb); tsdbEndFSTxn(pTsdb);
tsdbMemTableDestroy(pTsdb->imem); tsdbMemTableDestroy(pTsdb->imem);
pTsdb->imem = NULL; pTsdb->imem = NULL;
tsdbInfo("vgId:%d, commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); tsdbInfo("vgId:%d, commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
return code;
} }
static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) { static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) {
...@@ -354,34 +408,73 @@ static void tsdbDestroyCommitH(SCommitH *pCommith) { ...@@ -354,34 +408,73 @@ static void tsdbDestroyCommitH(SCommitH *pCommith) {
tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith));
} }
static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { static int32_t tsdbCommitToFileStart(SCommitH *pCHandle, SDFileSet *pSet, int32_t fid) {
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); int32_t code = 0;
STsdb *pRepo = TSDB_COMMIT_REPO(pCHandle);
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
ASSERT(pSet == NULL || pSet->fid == fid); ASSERT(pSet == NULL || pSet->fid == fid);
tsdbResetCommitFile(pCommith); pCHandle->fid = fid;
tsdbGetFidKeyRange(pCfg->days, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); pCHandle->pSet = pSet;
pCHandle->isRFileSet = false;
pCHandle->isDFileSame = false;
pCHandle->isLFileSame = false;
taosArrayClear(pCHandle->aBlkIdx);
tsdbGetFidKeyRange(pCfg->days, pCfg->precision, fid, &(pCHandle->minKey), &(pCHandle->maxKey));
code = tsdbSetAndOpenCommitFile(pCHandle, pSet, fid);
// Set and open files return code;
if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { }
static int32_t tsdbCommitToFileImpl(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitToFileEnd(SCommitH *pCommith) {
int32_t code = 0;
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith);
if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) <
0) {
tsdbError("vgId:%d, failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), pCommith->fid,
tstrerror(terrno));
tsdbCloseCommitFile(pCommith, true);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pCommith->pSet);
return -1; return -1;
} }
#if 0
// Loop to commit each table data
for (int tid = 0; tid < pCommith->niters; tid++) {
SCommitIter *pIter = pCommith->iters + tid;
if (pIter->pTable == NULL) continue; if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) {
tsdbError("vgId:%d, failed to update FSET %d header since %s", REPO_ID(pRepo), pCommith->fid, tstrerror(terrno));
tsdbCloseCommitFile(pCommith, true);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pCommith->pSet);
return -1;
}
if (tsdbCommitToTable(pCommith, tid) < 0) { // Close commit file
tsdbCloseCommitFile(pCommith, true); tsdbCloseCommitFile(pCommith, false);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) {
return -1; return -1;
}
} }
#endif
return code;
}
static int32_t tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
int32_t code = 0;
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith);
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
// commit to file start
code = tsdbCommitToFileStart(pCommith, pSet, fid);
if (code) {
goto _err;
}
// Loop to commit each table data in mem and file // Loop to commit each table data in mem and file
int mIter = 0, fIter = 0; int mIter = 0, fIter = 0;
int nBlkIdx = taosArrayGetSize(pCommith->readh.aBlkIdx); int nBlkIdx = taosArrayGetSize(pCommith->readh.aBlkIdx);
...@@ -426,31 +519,16 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { ...@@ -426,31 +519,16 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
} }
} }
if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < // commit to file end
0) { code = tsdbCommitToFileEnd(pCommith);
tsdbError("vgId:%d, failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); if (code) {
tsdbCloseCommitFile(pCommith, true); goto _err;
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
return -1;
}
if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) {
tsdbError("vgId:%d, failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
tsdbCloseCommitFile(pCommith, true);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
return -1;
} }
// Close commit file return code;
tsdbCloseCommitFile(pCommith, false);
if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) {
return -1;
}
return 0; _err:
return code;
} }
static int32_t tsdbCreateCommitIters(SCommitH *pCommith) { static int32_t tsdbCreateCommitIters(SCommitH *pCommith) {
...@@ -507,13 +585,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) { ...@@ -507,13 +585,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) {
pCommith->niters = 0; pCommith->niters = 0;
} }
static void tsdbResetCommitFile(SCommitH *pCommith) {
pCommith->isRFileSet = false;
pCommith->isDFileSame = false;
pCommith->isLFileSame = false;
taosArrayClear(pCommith->aBlkIdx);
}
static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
SDiskID did; SDiskID did;
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdb *pRepo = TSDB_COMMIT_REPO(pCommith);
...@@ -1590,4 +1661,171 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p ...@@ -1590,4 +1661,171 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
} }
return false; return false;
}
static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
bool merge) {
if (pCols) {
if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
*ppSchema = tsdbGetTableSchemaImpl(pTsdb, pTable, false, false, TD_ROW_SVER(row));
if (*ppSchema == NULL) {
ASSERT(false);
return -1;
}
}
tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
}
return 0;
}
static int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup,
SMergeInfo *pMergeInfo) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
TSKEY rowKey = 0;
TSKEY fKey = 0;
// only fetch lastKey from mem data as file data not used in this function actually
TSKEY lastKey = TSKEY_INITIAL_VAL;
bool isRowDel = false;
int filterIter = 0;
STSRow *row = NULL;
SMergeInfo mInfo;
// TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by
// query handle)
if (pMergeInfo == NULL) pMergeInfo = &mInfo;
memset(pMergeInfo, 0, sizeof(*pMergeInfo));
pMergeInfo->keyFirst = INT64_MAX;
pMergeInfo->keyLast = INT64_MIN;
if (pCols) tdResetDataCols(pCols);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
// 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols
// 2. rowKey - would dup since Multi-Version supported
while (true) {
if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
if (fKey < rowKey) {
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
#if 1
} else if (fKey > rowKey) {
if (isRowDel) {
// TODO: support delete function
pMergeInfo->rowsDeleteFailed++;
} else {
if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
if (lastKey != rowKey) {
pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
if (pCols) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pCols->numOfRows;
}
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
}
lastKey = rowKey;
} else {
if (keepDup) {
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
} else {
// discard
}
}
}
tsdbTbDataIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
} else { // fkey == rowKey
if (isRowDel) { // TODO: support delete function(How to stands for delete in file? rowVersion = -1?)
ASSERT(!keepDup);
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsDeleteSucceed++;
pMergeInfo->nOperations++;
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
} else {
if (keepDup) {
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
if (lastKey != rowKey) {
pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
if (pCols) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pCols->numOfRows;
}
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
}
lastKey = rowKey;
} else {
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
}
} else {
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
}
}
tsdbTbDataIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
}
#endif
}
if (pCols && (lastKey != TSKEY_INITIAL_VAL)) {
++pCols->numOfRows;
}
return 0;
} }
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbCommit2.c 已删除 100644 → 0
浏览文件 @ d5a29fc4
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
typedef struct {
SMemTable2 *pMemTable;
int32_t minutes;
int8_t precision;
TSKEY nCommitKey;
int32_t fid;
TSKEY minKey;
TSKEY maxKey;
SReadH readh;
SDFileSet wSet;
SArray *aBlkIdx;
SArray *aSupBlk;
SArray *aSubBlk;
SArray *aDelInfo;
} SCommitH;
static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb);
static int32_t tsdbCommitEnd(SCommitH *pCHandle);
static int32_t tsdbCommitImpl(SCommitH *pCHandle);
int32_t tsdbBegin2(STsdb *pTsdb) {
int32_t code = 0;
ASSERT(pTsdb->mem == NULL);
code = tsdbMemTableCreate2(pTsdb, (SMemTable2 **)&pTsdb->mem);
if (code) {
tsdbError("vgId:%d failed to begin TSDB since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
goto _exit;
}
_exit:
return code;
}
int32_t tsdbCommit2(STsdb *pTsdb) {
int32_t code = 0;
SCommitH ch = {0};
// start to commit
code = tsdbCommitStart(&ch, pTsdb);
if (code) {
goto _exit;
}
// commit
code = tsdbCommitImpl(&ch);
if (code) {
goto _err;
}
// end commit
code = tsdbCommitEnd(&ch);
if (code) {
goto _exit;
}
_exit:
return code;
_err:
tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb) {
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem;
tsdbInfo("vgId:%d start to commit", TD_VID(pTsdb->pVnode));
// switch to commit
ASSERT(pTsdb->imem == NULL && pTsdb->mem);
pTsdb->imem = pTsdb->mem;
pTsdb->mem = NULL;
// open handle
pCHandle->pMemTable = pMemTable;
pCHandle->minutes = pTsdb->keepCfg.days;
pCHandle->precision = pTsdb->keepCfg.precision;
pCHandle->nCommitKey = pMemTable->minKey.ts;
code = tsdbInitReadH(&pCHandle->readh, pTsdb);
if (code) {
goto _err;
}
pCHandle->aBlkIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCHandle->aBlkIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pCHandle->aSupBlk = taosArrayInit(0, sizeof(SBlock));
if (pCHandle->aSupBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pCHandle->aSubBlk = taosArrayInit(0, sizeof(SBlock));
if (pCHandle->aSubBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pCHandle->aDelInfo = taosArrayInit(0, sizeof(SDelInfo));
if (pCHandle->aDelInfo == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
// start FS transaction
tsdbStartFSTxn(pTsdb, 0, 0);
return code;
_err:
return code;
}
static int32_t tsdbCommitEnd(SCommitH *pCHandle) {
int32_t code = 0;
STsdb *pTsdb = pCHandle->pMemTable->pTsdb;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->imem;
// end transaction
code = tsdbEndFSTxn(pTsdb);
if (code) {
goto _err;
}
// close handle
taosArrayClear(pCHandle->aDelInfo);
taosArrayClear(pCHandle->aSubBlk);
taosArrayClear(pCHandle->aSupBlk);
taosArrayClear(pCHandle->aBlkIdx);
tsdbDestroyReadH(&pCHandle->readh);
// destroy memtable (todo: unref it)
pTsdb->imem = NULL;
tsdbMemTableDestroy2(pMemTable);
tsdbInfo("vgId:%d commit over", TD_VID(pTsdb->pVnode));
return code;
_err:
return code;
}
static int32_t tsdbCommitTableStart(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitTableEnd(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitTable(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) {
int32_t code = 0;
SMemDataIter iter = {0};
// commit table start
code = tsdbCommitTableStart(pCHandle);
if (code) {
goto _err;
}
// commit table impl
if (pMemData && pBlockIdx) {
// TODO
} else if (pMemData) {
// TODO
} else {
// TODO
}
// commit table end
code = tsdbCommitTableEnd(pCHandle);
if (code) {
goto _err;
}
return code;
_err:
return code;
}
static int32_t tsdbTableIdCmprFn(const void *p1, const void *p2) {
TABLEID *pId1 = (TABLEID *)p1;
TABLEID *pId2 = (TABLEID *)p2;
if (pId1->suid < pId2->suid) {
return -1;
} else if (pId1->suid > pId2->suid) {
return 1;
}
if (pId1->uid < pId2->uid) {
return -1;
} else if (pId1->uid > pId2->uid) {
return 1;
}
return 0;
}
static int32_t tsdbWriteBlockIdx(SDFile *pFile, SArray *pArray, uint8_t **ppBuf) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitFileStart(SCommitH *pCHandle) {
int32_t code = 0;
STsdb *pTsdb = pCHandle->pMemTable->pTsdb;
SDFileSet *pSet = NULL;
taosArrayClear(pCHandle->aBlkIdx);
return code;
}
static int32_t tsdbCommitFileEnd(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitFile(SCommitH *pCHandle) {
int32_t code = 0;
SMemData *pMemData;
SBlockIdx *pBlockIdx;
int32_t iMemData;
int32_t nMemData;
int32_t iBlockIdx;
int32_t nBlockIdx;
// commit file start
code = tsdbCommitFileStart(pCHandle);
if (code) {
goto _err;
}
// commit file impl
iMemData = 0;
nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData);
iBlockIdx = 0;
nBlockIdx = 0; // todo
for (;;) {
if (iMemData >= nMemData && iBlockIdx >= nBlockIdx) break;
pMemData = NULL;
pBlockIdx = NULL;
if (iMemData < nMemData) {
pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData);
}
if (iBlockIdx < nBlockIdx) {
// pBlockIdx = ;
}
if (pMemData && pBlockIdx) {
int32_t c = tsdbTableIdCmprFn(pMemData, pBlockIdx);
if (c < 0) {
iMemData++;
pBlockIdx = NULL;
} else if (c == 0) {
iMemData++;
iBlockIdx++;
} else {
iBlockIdx++;
pMemData = NULL;
}
} else {
if (pMemData) {
iMemData++;
} else {
iBlockIdx++;
}
}
code = tsdbCommitTable(pCHandle, pMemData, pBlockIdx);
if (code) {
goto _err;
}
}
// commit file end
code = tsdbCommitFileEnd(pCHandle);
if (code) {
goto _err;
}
return code;
_err:
return code;
}
static int32_t tsdbCommitData(SCommitH *pCHandle) {
int32_t code = 0;
int32_t fid;
if (pCHandle->pMemTable->nRows == 0) goto _exit;
// loop to commit to each file
for (;;) {
if (pCHandle->nCommitKey == TSKEY_MAX) break;
pCHandle->fid = TSDB_KEY_FID(pCHandle->nCommitKey, pCHandle->minutes, pCHandle->precision);
tsdbGetFidKeyRange(pCHandle->minutes, pCHandle->precision, pCHandle->fid, &pCHandle->minKey, &pCHandle->maxKey);
code = tsdbCommitFile(pCHandle);
if (code) {
goto _err;
}
}
_exit:
return code;
_err:
return code;
}
static int32_t delInfoCmprFn(const void *p1, const void *p2) {
SDelInfo *pDelInfo1 = (SDelInfo *)p1;
SDelInfo *pDelInfo2 = (SDelInfo *)p2;
if (pDelInfo1->suid < pDelInfo2->suid) {
return -1;
} else if (pDelInfo1->suid > pDelInfo2->suid) {
return 1;
}
if (pDelInfo1->uid < pDelInfo2->uid) {
return -1;
} else if (pDelInfo1->uid > pDelInfo2->uid) {
return 1;
}
if (pDelInfo1->version < pDelInfo2->version) {
return -1;
} else if (pDelInfo1->version > pDelInfo2->version) {
return 1;
}
return 0;
}
static int32_t tsdbCommitDelete(SCommitH *pCHandle) {
int32_t code = 0;
SDelInfo delInfo;
SMemData *pMemData;
if (pCHandle->pMemTable->nDelOp == 0) goto _exit;
// load del array (todo)
// loop to append SDelInfo
for (int32_t iMemData = 0; iMemData < taosArrayGetSize(pCHandle->pMemTable->aMemData); iMemData++) {
pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData);
for (SDelOp *pDelOp = pMemData->delOpHead; pDelOp; pDelOp = pDelOp->pNext) {
delInfo = (SDelInfo){.suid = pMemData->suid,
.uid = pMemData->uid,
.version = pDelOp->version,
.sKey = pDelOp->sKey,
.eKey = pDelOp->eKey};
if (taosArrayPush(pCHandle->aDelInfo, &delInfo) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
}
}
taosArraySort(pCHandle->aDelInfo, delInfoCmprFn);
// write to new file
_exit:
return code;
_err:
return code;
}
static int32_t tsdbCommitCache(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitImpl(SCommitH *pCHandle) {
int32_t code = 0;
// commit data
code = tsdbCommitData(pCHandle);
if (code) {
goto _err;
}
// commit delete
code = tsdbCommitDelete(pCHandle);
if (code) {
goto _err;
}
// commit cache if need (todo)
if (0) {
code = tsdbCommitCache(pCHandle);
if (code) {
goto _err;
}
}
return code;
_err:
return code;
}
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbMemTable.c
浏览文件 @ e12576ed
...@@ -188,23 +188,6 @@ _err: ...@@ -188,23 +188,6 @@ _err:
return code; return code;
} }
static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
bool merge) {
if (pCols) {
if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
*ppSchema = tsdbGetTableSchemaImpl(pTsdb, pTable, false, false, TD_ROW_SVER(row));
if (*ppSchema == NULL) {
ASSERT(false);
return -1;
}
}
tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
}
return 0;
}
int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter) { int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter) {
int32_t code = 0; int32_t code = 0;
...@@ -310,166 +293,6 @@ bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow) { ...@@ -310,166 +293,6 @@ bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow) {
return true; return true;
} }
/**
* This is an important function to load data or try to load data from memory skiplist iterator.
*
* This function load memory data until:
* 1. iterator ends
* 2. data key exceeds maxKey
* 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
* 4. operations in pCols not exceeds its max capacity if pCols is given
*
* The function tries to procceed AS MUCH AS POSSIBLE.
*/
int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
TSKEY rowKey = 0;
TSKEY fKey = 0;
// only fetch lastKey from mem data as file data not used in this function actually
TSKEY lastKey = TSKEY_INITIAL_VAL;
bool isRowDel = false;
int filterIter = 0;
STSRow *row = NULL;
SMergeInfo mInfo;
// TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by
// query handle)
if (pMergeInfo == NULL) pMergeInfo = &mInfo;
memset(pMergeInfo, 0, sizeof(*pMergeInfo));
pMergeInfo->keyFirst = INT64_MAX;
pMergeInfo->keyLast = INT64_MIN;
if (pCols) tdResetDataCols(pCols);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
// 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols
// 2. rowKey - would dup since Multi-Version supported
while (true) {
if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
if (fKey < rowKey) {
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
#if 1
} else if (fKey > rowKey) {
if (isRowDel) {
// TODO: support delete function
pMergeInfo->rowsDeleteFailed++;
} else {
if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
if (lastKey != rowKey) {
pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
if (pCols) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pCols->numOfRows;
}
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
}
lastKey = rowKey;
} else {
if (keepDup) {
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
} else {
// discard
}
}
}
tsdbTbDataIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
} else { // fkey == rowKey
if (isRowDel) { // TODO: support delete function(How to stands for delete in file? rowVersion = -1?)
ASSERT(!keepDup);
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsDeleteSucceed++;
pMergeInfo->nOperations++;
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
} else {
if (keepDup) {
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
if (lastKey != rowKey) {
pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
if (pCols) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pCols->numOfRows;
}
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
}
lastKey = rowKey;
} else {
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
}
} else {
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
}
}
tsdbTbDataIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
}
#endif
}
if (pCols && (lastKey != TSKEY_INITIAL_VAL)) {
++pCols->numOfRows;
}
return 0;
}
static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) { static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
int32_t code = 0; int32_t code = 0;
int32_t idx = 0; int32_t idx = 0;
......
source/dnode/vnode/src/tsdb/tsdbMemTable2.c 已删除 100644 → 0
浏览文件 @ d5a29fc4
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
typedef struct {
tb_uid_t uid;
STSchema *pTSchema;
} SSkmInfo;
#define SL_MAX_LEVEL 5
#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
#define SL_NODE_FORWARD(n, l) ((n)->forwards[l])
#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level))
#define SL_MOVE_BACKWARD 0x1
#define SL_MOVE_FROM_POS 0x2
static int32_t tsdbGetOrCreateMemData(SMemTable2 *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData);
static int memDataPCmprFn(const void *p1, const void *p2);
static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl);
static int32_t tsdbInsertTableDataImpl(SMemTable2 *pMemTable, SMemData *pMemData, int64_t version,
SVSubmitBlk *pSubmitBlk);
static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags);
// SMemTable ==============================================
int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable) {
int32_t code = 0;
SMemTable2 *pMemTable = NULL;
pMemTable = (SMemTable2 *)taosMemoryCalloc(1, sizeof(*pMemTable));
if (pMemTable == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pMemTable->pTsdb = pTsdb;
pMemTable->nRef = 1;
pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
pMemTable->nRows = 0;
pMemTable->nDelOp = 0;
pMemTable->aMemData = taosArrayInit(512, sizeof(SMemData *));
if (pMemTable->aMemData == NULL) {
taosMemoryFree(pMemTable);
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*ppMemTable = pMemTable;
return code;
_err:
*ppMemTable = NULL;
return code;
}
void tsdbMemTableDestroy2(SMemTable2 *pMemTable) {
taosArrayDestroyEx(pMemTable->aMemData, NULL /*TODO*/);
taosMemoryFree(pMemTable);
}
int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) {
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem; // TODO
SMemData *pMemData;
TSDBROW row = {.version = version};
ASSERT(pMemTable);
ASSERT(pSubmitBlk->nData > 0);
{
// check if table exists (todo)
}
code = tsdbGetOrCreateMemData(pMemTable, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData);
if (code) {
tsdbError("vgId:%d, failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
goto _err;
}
// do insert
code = tsdbInsertTableDataImpl(pMemTable, pMemData, version, pSubmitBlk);
if (code) {
goto _err;
}
return code;
_err:
return code;
}
int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) {
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem; // TODO
SMemData *pMemData;
SVBufPool *pPool = pTsdb->pVnode->inUse;
ASSERT(pMemTable);
{
// check if table exists (todo)
}
code = tsdbGetOrCreateMemData(pMemTable, suid, uid, &pMemData);
if (code) {
goto _err;
}
// do delete
SDelOp *pDelOp = (SDelOp *)vnodeBufPoolMalloc(pPool, sizeof(*pDelOp));
if (pDelOp == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pDelOp->version = version;
pDelOp->sKey = sKey;
pDelOp->eKey = eKey;
pDelOp->pNext = NULL;
if (pMemData->delOpHead == NULL) {
ASSERT(pMemData->delOpTail == NULL);
pMemData->delOpHead = pMemData->delOpTail = pDelOp;
} else {
pMemData->delOpTail->pNext = pDelOp;
pMemData->delOpTail = pDelOp;
}
{
// update the state of pMemTable, pMemData, last and lastrow (todo)
}
pMemTable->nDelOp++;
tsdbDebug("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
" since %s",
TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
return code;
_err:
tsdbError("vgId:%d, failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
" since %s",
TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
return code;
}
void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter) {
SMemSkipListNode *pos[SL_MAX_LEVEL];
pIter->pMemData = pMemData;
pIter->backward = backward;
pIter->pRow = NULL;
if (pKey == NULL) {
// create from head or tail
if (backward) {
pIter->pNode = SL_NODE_BACKWARD(pMemData->sl.pTail, 0);
} else {
pIter->pNode = SL_NODE_FORWARD(pMemData->sl.pHead, 0);
}
} else {
// create from a key
if (backward) {
memDataMovePosTo(pMemData, pos, pKey, SL_MOVE_BACKWARD);
pIter->pNode = SL_NODE_BACKWARD(pos[0], 0);
} else {
memDataMovePosTo(pMemData, pos, pKey, 0);
pIter->pNode = SL_NODE_FORWARD(pos[0], 0);
}
}
}
bool tsdbMemDataIterNext(SMemDataIter *pIter) {
SMemSkipListNode *pHead = pIter->pMemData->sl.pHead;
SMemSkipListNode *pTail = pIter->pMemData->sl.pTail;
pIter->pRow = NULL;
if (pIter->backward) {
ASSERT(pIter->pNode != pTail);
if (pIter->pNode == pHead) {
return false;
}
pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0);
if (pIter->pNode == pHead) {
return false;
}
} else {
ASSERT(pIter->pNode != pHead);
if (pIter->pNode == pTail) {
return false;
}
pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0);
if (pIter->pNode == pTail) {
return false;
}
}
return true;
}
void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow) {
if (pIter->pRow) {
*ppRow = pIter->pRow;
} else {
SMemSkipListNode *pHead = pIter->pMemData->sl.pHead;
SMemSkipListNode *pTail = pIter->pMemData->sl.pTail;
if (pIter->backward) {
ASSERT(pIter->pNode != pTail);
if (pIter->pNode == pHead) {
*ppRow = NULL;
} else {
tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
*ppRow = &pIter->row;
}
} else {
ASSERT(pIter->pNode != pHead);
if (pIter->pNode == pTail) {
*ppRow = NULL;
} else {
tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
*ppRow = &pIter->row;
}
}
}
}
static int32_t tsdbGetOrCreateMemData(SMemTable2 *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) {
int32_t code = 0;
int32_t idx = 0;
SMemData *pMemDataT = &(SMemData){.suid = suid, .uid = uid};
SMemData *pMemData = NULL;
SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
// get
idx = taosArraySearchIdx(pMemTable->aMemData, &pMemDataT, memDataPCmprFn, TD_GE);
if (idx >= 0) {
pMemData = (SMemData *)taosArrayGet(pMemTable->aMemData, idx);
if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit;
}
// create
pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData) + SL_NODE_SIZE(maxLevel) * 2);
if (pMemData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pMemData->suid = suid;
pMemData->uid = uid;
pMemData->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
pMemData->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
pMemData->delOpHead = pMemData->delOpTail = NULL;
pMemData->sl.seed = taosRand();
pMemData->sl.size = 0;
pMemData->sl.maxLevel = maxLevel;
pMemData->sl.level = 0;
pMemData->sl.pHead = (SMemSkipListNode *)&pMemData[1];
pMemData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pMemData->sl.pHead, SL_NODE_SIZE(maxLevel));
pMemData->sl.pHead->level = maxLevel;
pMemData->sl.pTail->level = maxLevel;
for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) {
SL_NODE_FORWARD(pMemData->sl.pHead, iLevel) = pMemData->sl.pTail;
SL_NODE_BACKWARD(pMemData->sl.pHead, iLevel) = NULL;
SL_NODE_BACKWARD(pMemData->sl.pTail, iLevel) = pMemData->sl.pHead;
SL_NODE_FORWARD(pMemData->sl.pTail, iLevel) = NULL;
}
if (idx < 0) idx = 0;
if (taosArrayInsert(pMemTable->aMemData, idx, &pMemData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
_exit:
*ppMemData = pMemData;
return code;
_err:
*ppMemData = NULL;
return code;
}
static int memDataPCmprFn(const void *p1, const void *p2) {
SMemData *pMemData1 = *(SMemData **)p1;
SMemData *pMemData2 = *(SMemData **)p2;
if (pMemData1->suid < pMemData2->suid) {
return -1;
} else if (pMemData1->suid > pMemData2->suid) {
return 1;
}
if (pMemData1->uid < pMemData2->uid) {
return -1;
} else if (pMemData1->uid > pMemData2->uid) {
return 1;
}
return 0;
}
static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) {
int32_t n = 0;
n += tPutI64(p ? p + n : p, pRow->version);
n += tPutTSRow(p ? p + n : p, &pRow->tsRow);
return n;
}
static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) {
int32_t n = 0;
n += tGetI64(p + n, &pRow->version);
n += tGetTSRow(p + n, &pRow->tsRow);
return n;
}
static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
int8_t level = 1;
int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
const uint32_t factor = 4;
while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
level++;
}
return level;
}
static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags) {
SMemSkipListNode *px;
SMemSkipListNode *pn;
TSDBKEY *pTKey;
int c;
int backward = flags & SL_MOVE_BACKWARD;
int fromPos = flags & SL_MOVE_FROM_POS;
if (backward) {
px = pMemData->sl.pTail;
for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) {
pos[iLevel] = px;
}
if (pMemData->sl.level) {
if (fromPos) px = pos[pMemData->sl.level - 1];
for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) {
pn = SL_NODE_BACKWARD(px, iLevel);
while (pn != pMemData->sl.pHead) {
pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
c = tsdbKeyCmprFn(pTKey, pKey);
if (c <= 0) {
break;
} else {
px = pn;
pn = SL_NODE_BACKWARD(px, iLevel);
}
}
pos[iLevel] = px;
}
}
} else {
px = pMemData->sl.pHead;
for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) {
pos[iLevel] = px;
}
if (pMemData->sl.level) {
if (fromPos) px = pos[pMemData->sl.level - 1];
for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) {
pn = SL_NODE_FORWARD(px, iLevel);
while (pn != pMemData->sl.pHead) {
pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
c = tsdbKeyCmprFn(pTKey, pKey);
if (c >= 0) {
break;
} else {
px = pn;
pn = SL_NODE_FORWARD(px, iLevel);
}
}
pos[iLevel] = px;
}
}
}
}
static int32_t memDataDoPut(SMemTable2 *pMemTable, SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow,
int8_t forward) {
int32_t code = 0;
int8_t level;
SMemSkipListNode *pNode;
SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
// node
level = tsdbMemSkipListRandLevel(&pMemData->sl);
pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow));
if (pNode == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pNode->level = level;
for (int8_t iLevel = 0; iLevel < level; iLevel++) {
SL_NODE_FORWARD(pNode, iLevel) = NULL;
SL_NODE_BACKWARD(pNode, iLevel) = NULL;
}
tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow);
// put
for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) {
SMemSkipListNode *px = pos[iLevel];
if (forward) {
SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel);
SL_NODE_FORWARD(pNode, iLevel) = pNext;
SL_NODE_BACKWARD(pNode, iLevel) = px;
SL_NODE_BACKWARD(pNext, iLevel) = pNode;
SL_NODE_FORWARD(px, iLevel) = pNode;
} else {
SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel);
SL_NODE_FORWARD(pNode, iLevel) = px;
SL_NODE_BACKWARD(pNode, iLevel) = pPrev;
SL_NODE_FORWARD(pPrev, iLevel) = pNode;
SL_NODE_BACKWARD(px, iLevel) = pNode;
}
}
pMemData->sl.size++;
if (pMemData->sl.level < pNode->level) {
pMemData->sl.level = pNode->level;
}
_exit:
return code;
}
static int32_t tsdbInsertTableDataImpl(SMemTable2 *pMemTable, SMemData *pMemData, int64_t version,
SVSubmitBlk *pSubmitBlk) {
int32_t code = 0;
int32_t n = 0;
uint8_t *p = pSubmitBlk->pData;
int32_t nRow = 0;
TSDBROW row = {.version = version};
SMemSkipListNode *pos[SL_MAX_LEVEL];
ASSERT(pSubmitBlk->nData);
// backward put first data
n += tGetTSRow(p + n, &row.tsRow);
ASSERT(n <= pSubmitBlk->nData);
memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_BACKWARD);
code = memDataDoPut(pMemTable, pMemData, pos, &row, 0);
if (code) {
goto _exit;
}
nRow++;
if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->minKey) < 0) {
pMemData->minKey = *(TSDBKEY *)&row;
}
if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->minKey) < 0) {
pMemTable->minKey = *(TSDBKEY *)&row;
}
// forward put rest
for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) {
pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel);
}
while (n < pSubmitBlk->nData) {
n += tGetTSRow(p + n, &row.tsRow);
ASSERT(n <= pSubmitBlk->nData);
memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_FROM_POS);
code = memDataDoPut(pMemTable, pMemData, pos, &row, 1);
if (code) {
goto _exit;
}
nRow++;
}
if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->maxKey) > 0) {
pMemData->maxKey = *(TSDBKEY *)&row;
}
if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->maxKey) > 0) {
pMemTable->maxKey = *(TSDBKEY *)&row;
}
pMemTable->nRows += nRow;
_exit:
return code;
}
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
浏览文件 @ e12576ed
...@@ -196,33 +196,6 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) { ...@@ -196,33 +196,6 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
} else { } else {
pReadh->pBlkIdx = (SBlockIdx *)p; pReadh->pBlkIdx = (SBlockIdx *)p;
} }
// size_t size = taosArrayGetSize(pReadh->aBlkIdx);
// if (size > 0) {
// while (true) {
// if (pReadh->cidx >= size) {
// pReadh->pBlkIdx = NULL;
// break;
// }
// SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
// if (pBlkIdx->uid == TABLE_TID(pTable)) {
// if (pBlkIdx->uid == TABLE_UID(pTable)) {
// pReadh->pBlkIdx = pBlkIdx;
// } else {
// pReadh->pBlkIdx = NULL;
// }
// pReadh->cidx++;
// break;
// } else if (pBlkIdx->uid > TABLE_TID(pTable)) {
// pReadh->pBlkIdx = NULL;
// break;
// } else {
// pReadh->cidx++;
// }
// }
// } else {
// pReadh->pBlkIdx = NULL;
// }
return 0; return 0;
} }
......
source/libs/executor/inc/executorimpl.h
浏览文件 @ e12576ed
...@@ -353,6 +353,7 @@ typedef struct STagScanInfo { ...@@ -353,6 +353,7 @@ typedef struct STagScanInfo {
int32_t curPos; int32_t curPos;
SReadHandle readHandle; SReadHandle readHandle;
STableListInfo *pTableList; STableListInfo *pTableList;
SNode* pFilterNode; // filter info,
} STagScanInfo; } STagScanInfo;
typedef enum EStreamScanMode { typedef enum EStreamScanMode {
...@@ -747,7 +748,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan ...@@ -747,7 +748,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
void doSetOperatorCompleted(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree);
SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset);
void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols);
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
...@@ -777,7 +778,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR ...@@ -777,7 +778,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
......
source/libs/executor/src/executorimpl.c
浏览文件 @ e12576ed
...@@ -1818,9 +1818,9 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO ...@@ -1818,9 +1818,9 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
} }
} }
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree) {
if (pFilterNode == NULL) { if (pFilterNode == NULL) {
return; return;
} }
...@@ -1839,11 +1839,11 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { ...@@ -1839,11 +1839,11 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols);
filterFreeInfo(filter); filterFreeInfo(filter);
extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); extractQualifiedTupleByFilterResult(pBlock, rowRes, keep, needFree);
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
} }
void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree) {
if (keep) { if (keep) {
return; return;
} }
...@@ -1883,8 +1883,20 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR ...@@ -1883,8 +1883,20 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
ASSERT(pBlock->info.rows == numOfRows); ASSERT(pBlock->info.rows == numOfRows);
} }
SColumnInfoData tmp = *pSrc;
*pSrc = *pDst; *pSrc = *pDst;
*pDst = tmp;
if (!needFree) {
if (IS_VAR_DATA_TYPE(pDst->info.type)) { // this elements do not need free
pDst->varmeta.offset = NULL;
} else {
pDst->nullbitmap = NULL;
}
pDst->pData = NULL;
}
} }
blockDataDestroy(px); // fix memory leak
} else { } else {
// do nothing // do nothing
pBlock->info.rows = 0; pBlock->info.rows = 0;
...@@ -3640,7 +3652,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { ...@@ -3640,7 +3652,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
longjmp(pTaskInfo->env, code); longjmp(pTaskInfo->env, code);
} }
doFilter(pProjectInfo->pFilterNode, pBlock); doFilter(pProjectInfo->pFilterNode, pBlock, true);
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false); setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false);
blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
...@@ -4522,7 +4534,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo ...@@ -4522,7 +4534,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
STimeWindowAggSupp twSup = { STimeWindowAggSupp twSup = {
.waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
tsdbReaderT pDataReader = NULL; tsdbReaderT pDataReader = NULL;
if (pHandle->vnode) { if (pHandle->vnode) {
pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);
......
source/libs/executor/src/groupoperator.c
浏览文件 @ e12576ed
...@@ -359,7 +359,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { ...@@ -359,7 +359,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
while(1) { while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pRes); doFilter(pInfo->pCondition, pRes, true);
bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo); bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) { if (!hasRemain) {
......
source/libs/executor/src/scanoperator.c
浏览文件 @ e12576ed
...@@ -267,7 +267,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca ...@@ -267,7 +267,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
} }
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
doFilter(pTableScanInfo->pFilterNode, pBlock); doFilter(pTableScanInfo->pFilterNode, pBlock, false);
int64_t et = taosGetTimestampMs(); int64_t et = taosGetTimestampMs();
pTableScanInfo->readRecorder.filterTime += (et - st); pTableScanInfo->readRecorder.filterTime += (et - st);
...@@ -737,8 +737,8 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) { ...@@ -737,8 +737,8 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
SSDataBlock* pSDB = pInfo->pUpdateRes; SSDataBlock* pSDB = pInfo->pUpdateRes;
STimeWindow win = { STimeWindow win = {
.skey = INT64_MIN, .skey = INT64_MIN,
.ekey = INT64_MAX, .ekey = INT64_MAX,
}; };
bool needRead = false; bool needRead = false;
if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) { if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) {
...@@ -950,7 +950,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { ...@@ -950,7 +950,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes);
} }
doFilter(pInfo->pCondition, pInfo->pRes); doFilter(pInfo->pCondition, pInfo->pRes, false);
blockDataUpdateTsWindow(pInfo->pRes, 0); blockDataUpdateTsWindow(pInfo->pRes, 0);
break; break;
} }
...@@ -1722,7 +1722,9 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { ...@@ -1722,7 +1722,9 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
} }
pRes->info.rows = count; pRes->info.rows = count;
pOperator->resultInfo.totalRows += count; doFilter(pInfo->pFilterNode, pRes, true);
pOperator->resultInfo.totalRows += pRes->info.rows;
return (pRes->info.rows == 0) ? NULL : pInfo->pRes; return (pRes->info.rows == 0) ? NULL : pInfo->pRes;
} }
......
source/libs/function/inc/builtinsimpl.h
浏览文件 @ e12576ed
...@@ -23,9 +23,13 @@ extern "C" { ...@@ -23,9 +23,13 @@ extern "C" {
#include "function.h" #include "function.h"
#include "functionMgt.h" #include "functionMgt.h"
bool dummyGetEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* UNUSED_PARAM(pEnv));
bool dummyInit(SqlFunctionCtx* UNUSED_PARAM(pCtx), SResultRowEntryInfo* UNUSED_PARAM(pResultInfo));
int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx));
int32_t dummyFinalize(SqlFunctionCtx* UNUSED_PARAM(pCtx), SSDataBlock* UNUSED_PARAM(pBlock));
bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx));
int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult); int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
...@@ -74,10 +78,13 @@ bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultI ...@@ -74,10 +78,13 @@ bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultI
int32_t percentileFunction(SqlFunctionCtx *pCtx); int32_t percentileFunction(SqlFunctionCtx *pCtx);
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t getApercentileMaxSize();
bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool apercentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool apercentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t apercentileFunction(SqlFunctionCtx *pCtx); int32_t apercentileFunction(SqlFunctionCtx *pCtx);
int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx);
int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
...@@ -95,10 +102,13 @@ int32_t topFunction(SqlFunctionCtx *pCtx); ...@@ -95,10 +102,13 @@ int32_t topFunction(SqlFunctionCtx *pCtx);
int32_t bottomFunction(SqlFunctionCtx *pCtx); int32_t bottomFunction(SqlFunctionCtx *pCtx);
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t getSpreadInfoSize();
bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t spreadFunction(SqlFunctionCtx* pCtx); int32_t spreadFunction(SqlFunctionCtx* pCtx);
int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx);
int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
......
source/libs/function/src/builtins.c
浏览文件 @ e12576ed
...@@ -251,6 +251,72 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t ...@@ -251,6 +251,72 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateApercentileImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (isPartial) {
if (2 != numOfParams && 3 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
// param1
SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1);
if (nodeType(pParamNode1) != QUERY_NODE_VALUE) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
SValueNode* pValue = (SValueNode*)pParamNode1;
if (pValue->datum.i < 0 || pValue->datum.i > 100) {
return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
}
pValue->notReserved = true;
uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
if (!IS_NUMERIC_TYPE(para1Type) || !IS_INTEGER_TYPE(para2Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
// param2
if (3 == numOfParams) {
uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type;
if (!IS_VAR_DATA_TYPE(para3Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
SNode* pParamNode2 = nodesListGetNode(pFunc->pParameterList, 2);
if (QUERY_NODE_VALUE != nodeType(pParamNode2) || !validateApercentileAlgo((SValueNode*)pParamNode2)) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
"Third parameter algorithm of apercentile must be 'default' or 't-digest'");
}
pValue = (SValueNode*)pParamNode2;
pValue->notReserved = true;
}
pFunc->node.resType = (SDataType){.bytes = getApercentileMaxSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
} else {
if (1 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (TSDB_DATA_TYPE_BINARY != para1Type) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateApercentilePartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateApercentileImpl(pFunc, pErrBuf, len, true);
}
static int32_t translateApercentileMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateApercentileImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters // pseudo column do not need to check parameters
pFunc->node.resType = pFunc->node.resType =
...@@ -311,6 +377,34 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) ...@@ -311,6 +377,34 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateSpreadImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (isPartial) {
if (!IS_NUMERIC_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
} else {
if (TSDB_DATA_TYPE_BINARY != paraType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateSpreadPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateSpreadImpl(pFunc, pErrBuf, len, true);
}
static int32_t translateSpreadMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateSpreadImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (1 != numOfParams && 2 != numOfParams) { if (1 != numOfParams && 2 != numOfParams) {
...@@ -1056,9 +1150,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1056,9 +1150,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.processFunc = countFunction, .processFunc = countFunction,
.finalizeFunc = functionFinalize, .finalizeFunc = functionFinalize,
.invertFunc = countInvertFunction, .invertFunc = countInvertFunction,
.combineFunc = combineFunction, .combineFunc = combineFunction,
// .pPartialFunc = "count", .pPartialFunc = "count",
// .pMergeFunc = "sum" .pMergeFunc = "sum"
}, },
{ {
.name = "sum", .name = "sum",
...@@ -1071,7 +1165,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1071,7 +1165,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.processFunc = sumFunction, .processFunc = sumFunction,
.finalizeFunc = functionFinalize, .finalizeFunc = functionFinalize,
.invertFunc = sumInvertFunction, .invertFunc = sumInvertFunction,
.combineFunc = sumCombine, .combineFunc = sumCombine,
.pPartialFunc = "sum",
.pMergeFunc = "sum"
}, },
{ {
.name = "min", .name = "min",
...@@ -1083,7 +1179,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1083,7 +1179,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = minmaxFunctionSetup, .initFunc = minmaxFunctionSetup,
.processFunc = minFunction, .processFunc = minFunction,
.finalizeFunc = minmaxFunctionFinalize, .finalizeFunc = minmaxFunctionFinalize,
.combineFunc = minCombine .combineFunc = minCombine,
.pPartialFunc = "min",
.pMergeFunc = "min"
}, },
{ {
.name = "max", .name = "max",
...@@ -1095,7 +1193,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1095,7 +1193,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = minmaxFunctionSetup, .initFunc = minmaxFunctionSetup,
.processFunc = maxFunction, .processFunc = maxFunction,
.finalizeFunc = minmaxFunctionFinalize, .finalizeFunc = minmaxFunctionFinalize,
.combineFunc = maxCombine .combineFunc = maxCombine,
.pPartialFunc = "max",
.pMergeFunc = "max"
}, },
{ {
.name = "stddev", .name = "stddev",
...@@ -1151,6 +1251,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1151,6 +1251,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.getEnvFunc = getApercentileFuncEnv, .getEnvFunc = getApercentileFuncEnv,
.initFunc = apercentileFunctionSetup, .initFunc = apercentileFunctionSetup,
.processFunc = apercentileFunction, .processFunc = apercentileFunction,
.finalizeFunc = apercentileFinalize,
.pPartialFunc = "_apercentile_partial",
.pMergeFunc = "_apercentile_merge"
},
{
.name = "_apercentile_partial",
.type = FUNCTION_TYPE_APERCENTILE_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateApercentilePartial,
.getEnvFunc = getApercentileFuncEnv,
.initFunc = apercentileFunctionSetup,
.processFunc = apercentileFunction,
.finalizeFunc = apercentilePartialFinalize
},
{
.name = "_apercentile_merge",
.type = FUNCTION_TYPE_APERCENTILE_MERGE,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateApercentileMerge,
.getEnvFunc = getApercentileFuncEnv,
.initFunc = functionSetup,
.processFunc = apercentileFunctionMerge,
.finalizeFunc = apercentileFinalize .finalizeFunc = apercentileFinalize
}, },
{ {
...@@ -1182,6 +1304,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1182,6 +1304,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.getEnvFunc = getSpreadFuncEnv, .getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup, .initFunc = spreadFunctionSetup,
.processFunc = spreadFunction, .processFunc = spreadFunction,
.finalizeFunc = spreadFinalize,
.pPartialFunc = "_spread_partial",
.pMergeFunc = "_spread_merge"
},
{
.name = "_spread_partial",
.type = FUNCTION_TYPE_SPREAD_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateSpreadPartial,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup,
.processFunc = spreadFunction,
.finalizeFunc = spreadPartialFinalize
},
{
.name = "_spread_merge",
.type = FUNCTION_TYPE_SPREAD_MERGE,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateSpreadMerge,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup,
.processFunc = spreadFunctionMerge,
.finalizeFunc = spreadFinalize .finalizeFunc = spreadFinalize
}, },
{ {
...@@ -1214,7 +1360,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1214,7 +1360,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = functionSetup, .initFunc = functionSetup,
.processFunc = firstFunction, .processFunc = firstFunction,
.finalizeFunc = firstLastFinalize, .finalizeFunc = firstLastFinalize,
.combineFunc = firstCombine, .combineFunc = firstCombine,
}, },
{ {
.name = "last", .name = "last",
......
source/libs/function/src/builtinsimpl.c
浏览文件 @ e12576ed
...@@ -100,6 +100,7 @@ typedef struct SPercentileInfo { ...@@ -100,6 +100,7 @@ typedef struct SPercentileInfo {
typedef struct SAPercentileInfo { typedef struct SAPercentileInfo {
double result; double result;
double percent;
int8_t algo; int8_t algo;
SHistogramInfo *pHisto; SHistogramInfo *pHisto;
TDigest *pTDigest; TDigest *pTDigest;
...@@ -283,6 +284,22 @@ typedef struct SUniqueInfo { ...@@ -283,6 +284,22 @@ typedef struct SUniqueInfo {
} \ } \
} while (0) } while (0)
bool dummyGetEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* UNUSED_PARAM(pEnv)) {
return true;
}
bool dummyInit(SqlFunctionCtx* UNUSED_PARAM(pCtx), SResultRowEntryInfo* UNUSED_PARAM(pResultInfo)) {
return true;
}
int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx)) {
return 0;
}
int32_t dummyFinalize(SqlFunctionCtx* UNUSED_PARAM(pCtx), SSDataBlock* UNUSED_PARAM(pBlock)) {
return 0;
}
bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
if (pResultInfo->initialized) { if (pResultInfo->initialized) {
return false; return false;
...@@ -327,10 +344,6 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { ...@@ -327,10 +344,6 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx)) {
return 0;
}
int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult) { int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId; int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
...@@ -602,7 +615,7 @@ int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { ...@@ -602,7 +615,7 @@ int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
SSumRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); SSumRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
pDBuf->isum += pSBuf->isum; pDBuf->isum += pSBuf->isum;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
...@@ -1974,6 +1987,12 @@ bool getApercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { ...@@ -1974,6 +1987,12 @@ bool getApercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
return true; return true;
} }
int32_t getApercentileMaxSize() {
int32_t bytesHist = (int32_t)(sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
int32_t bytesDigest = (int32_t)(sizeof(SAPercentileInfo) + TDIGEST_SIZE(COMPRESSION));
return TMAX(bytesHist, bytesDigest);
}
static int8_t getApercentileAlgo(char *algoStr) { static int8_t getApercentileAlgo(char *algoStr) {
int8_t algoType; int8_t algoType;
if (strcasecmp(algoStr, "default") == 0) { if (strcasecmp(algoStr, "default") == 0) {
...@@ -1988,16 +2007,24 @@ static int8_t getApercentileAlgo(char *algoStr) { ...@@ -1988,16 +2007,24 @@ static int8_t getApercentileAlgo(char *algoStr) {
} }
static void buildHistogramInfo(SAPercentileInfo* pInfo) { static void buildHistogramInfo(SAPercentileInfo* pInfo) {
pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); pInfo->pHisto = (SHistogramInfo*) ((char*)pInfo + sizeof(SAPercentileInfo));
pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo));
} }
static void buildTDigestInfo(SAPercentileInfo* pInfo) {
pInfo->pTDigest = (TDigest*)((char*)pInfo + sizeof(SAPercentileInfo));
}
bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) { if (!functionSetup(pCtx, pResultInfo)) {
return false; return false;
} }
SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
SVariant* pVal = &pCtx->param[1].param;
pInfo->percent = (pVal->nType == TSDB_DATA_TYPE_BIGINT) ? pVal->i : pVal->d;
if (pCtx->numOfParams == 2) { if (pCtx->numOfParams == 2) {
pInfo->algo = APERCT_ALGO_DEFAULT; pInfo->algo = APERCT_ALGO_DEFAULT;
} else if (pCtx->numOfParams == 3) { } else if (pCtx->numOfParams == 3) {
...@@ -2062,23 +2089,87 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { ...@@ -2062,23 +2089,87 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) {
SVariant* pVal = &pCtx->param[1].param; int32_t numOfElems = 0;
double percent = (pVal->nType == TSDB_DATA_TYPE_BIGINT) ? pVal->i : pVal->d; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SAPercentileInfo* pInputInfo;
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < pInput->numOfRows + start; ++i) {
//if (colDataIsNull_s(pCol, i)) {
// continue;
//}
numOfElems += 1;
char* data = colDataGetData(pCol, i);
pInputInfo = (SAPercentileInfo *)varDataVal(data);
}
pInfo->percent = pInputInfo->percent;
pInfo->algo = pInputInfo->algo;
if (pInfo->algo == APERCT_ALGO_TDIGEST) {
buildTDigestInfo(pInputInfo);
tdigestAutoFill(pInputInfo->pTDigest, COMPRESSION);
if(pInputInfo->pTDigest->num_centroids == 0 && pInputInfo->pTDigest->num_buffered_pts == 0) {
return TSDB_CODE_SUCCESS;
}
buildTDigestInfo(pInfo);
TDigest *pTDigest = pInfo->pTDigest;
if(pTDigest->num_centroids <= 0) {
memcpy(pTDigest, pInputInfo->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION));
tdigestAutoFill(pTDigest, COMPRESSION);
} else {
tdigestMerge(pTDigest, pInputInfo->pTDigest);
}
} else {
buildHistogramInfo(pInputInfo);
if (pInputInfo->pHisto->numOfElems <= 0) {
return TSDB_CODE_SUCCESS;
}
buildHistogramInfo(pInfo);
SHistogramInfo *pHisto = pInfo->pHisto;
if (pHisto->numOfElems <= 0) {
memcpy(pHisto, pInputInfo->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
} else {
pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
SHistogramInfo *pRes = tHistogramMerge(pHisto, pInputInfo->pHisto, MAX_HISTOGRAM_BIN);
memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN);
pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
tHistogramDestroy(&pRes);
}
}
SET_VAL(pResInfo, numOfElems, 1);
return TSDB_CODE_SUCCESS;
}
int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SAPercentileInfo* pInfo = (SAPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo); SAPercentileInfo* pInfo = (SAPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->algo == APERCT_ALGO_TDIGEST) { if (pInfo->algo == APERCT_ALGO_TDIGEST) {
if (pInfo->pTDigest->size > 0) { if (pInfo->pTDigest->size > 0) {
pInfo->result = tdigestQuantile(pInfo->pTDigest, percent/100); pInfo->result = tdigestQuantile(pInfo->pTDigest, pInfo->percent / 100);
} else { // no need to free } else { // no need to free
//setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); //setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} else { } else {
if (pInfo->pHisto->numOfElems > 0) { if (pInfo->pHisto->numOfElems > 0) {
double ratio[] = {percent}; double ratio[] = {pInfo->percent};
double *res = tHistogramUniform(pInfo->pHisto, ratio, 1); double *res = tHistogramUniform(pInfo->pHisto, ratio, 1);
pInfo->result = *res; pInfo->result = *res;
//memcpy(pCtx->pOutput, res, sizeof(double)); //memcpy(pCtx->pOutput, res, sizeof(double));
...@@ -2092,6 +2183,40 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { ...@@ -2092,6 +2183,40 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return functionFinalize(pCtx, pBlock); return functionFinalize(pCtx, pBlock);
} }
int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SAPercentileInfo* pInfo = (SAPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo);
int32_t bytesHist = (int32_t)(sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
int32_t bytesDigest = (int32_t)(sizeof(SAPercentileInfo) + TDIGEST_SIZE(COMPRESSION));
int32_t resultBytes = TMAX(bytesHist, bytesDigest);
char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
if (pInfo->algo == APERCT_ALGO_TDIGEST) {
if (pInfo->pTDigest->size > 0) {
memcpy(varDataVal(res), pInfo, resultBytes);
varDataSetLen(res, resultBytes);
} else {
return TSDB_CODE_SUCCESS;
}
} else {
if (pInfo->pHisto->numOfElems > 0) {
memcpy(varDataVal(res), pInfo, resultBytes);
varDataSetLen(res, resultBytes);
} else {
return TSDB_CODE_SUCCESS;
}
}
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
colDataAppend(pCol, pBlock->info.rows, res, false);
taosMemoryFree(res);
return pResInfo->numOfRes;
}
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0);
pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t); pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t);
...@@ -2743,6 +2868,10 @@ bool getSpreadFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { ...@@ -2743,6 +2868,10 @@ bool getSpreadFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true; return true;
} }
int32_t getSpreadInfoSize() {
return (int32_t)sizeof(SSpreadInfo);
}
bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) { if (!functionSetup(pCtx, pResultInfo)) {
return false; return false;
...@@ -2828,6 +2957,32 @@ _spread_over: ...@@ -2828,6 +2957,32 @@ _spread_over:
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t spreadFunctionMerge(SqlFunctionCtx *pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
SSpreadInfo* pInputInfo;
int32_t start = pInput->startRowIndex;
char* data = colDataGetData(pCol, start);
pInputInfo = (SSpreadInfo *)varDataVal(data);
pInfo->hasResult = pInputInfo->hasResult;
if (pInputInfo->max > pInfo->max) {
pInfo->max = pInputInfo->max;
}
if (pInputInfo->min < pInfo->min) {
pInfo->min = pInputInfo->min;
}
SET_VAL(GET_RES_INFO(pCtx), 1, 1);
return TSDB_CODE_SUCCESS;
}
int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (pInfo->hasResult == true) { if (pInfo->hasResult == true) {
...@@ -2836,6 +2991,24 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { ...@@ -2836,6 +2991,24 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return functionFinalize(pCtx, pBlock); return functionFinalize(pCtx, pBlock);
} }
int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = (int32_t)sizeof(SSpreadInfo);
char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
memcpy(varDataVal(res), pInfo, resultBytes);
varDataSetLen(res, resultBytes);
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
colDataAppend(pCol, pBlock->info.rows, res, false);
taosMemoryFree(res);
return pResInfo->numOfRes;
}
bool getElapsedFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { bool getElapsedFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SElapsedInfo); pEnv->calcMemSize = sizeof(SElapsedInfo);
return true; return true;
......
source/libs/index/inc/indexComm.h
浏览文件 @ e12576ed
...@@ -28,6 +28,7 @@ extern char JSON_VALUE_DELIM; ...@@ -28,6 +28,7 @@ extern char JSON_VALUE_DELIM;
char* indexPackJsonData(SIndexTerm* itm); char* indexPackJsonData(SIndexTerm* itm);
char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
char* indexPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip);
typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
......
source/libs/index/inc/indexInt.h
浏览文件 @ e12576ed
...@@ -43,7 +43,7 @@ extern "C" { ...@@ -43,7 +43,7 @@ extern "C" {
#define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0) #define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0)
// clang-format on // clang-format on
typedef enum { LT, LE, GT, GE, CONTAINS } RangeType; typedef enum { LT, LE, GT, GE, CONTAINS, EQ } RangeType;
typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef enum { kTypeValue, kTypeDeletion } STermValueType;
typedef struct SIndexStat { typedef struct SIndexStat {
......
source/libs/index/src/indexCache.c
浏览文件 @ e12576ed
...@@ -48,6 +48,7 @@ static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STer ...@@ -48,6 +48,7 @@ static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STer
/*comm func of compare, used in (LE/LT/GE/GT compare)*/ /*comm func of compare, used in (LE/LT/GE/GT compare)*/
static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s, RangeType type); static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s, RangeType type);
static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
static int32_t cacheSearchEqual_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
...@@ -63,7 +64,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR ...@@ -63,7 +64,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s) = { static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s) = {
{cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual, {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual,
cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange},
{cacheSearchTerm_JSON, cacheSearchPrefix_JSON, cacheSearchSuffix_JSON, cacheSearchRegex_JSON, {cacheSearchEqual_JSON, cacheSearchPrefix_JSON, cacheSearchSuffix_JSON, cacheSearchRegex_JSON,
cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON, cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON,
cacheSearchRange_JSON}}; cacheSearchRange_JSON}};
...@@ -123,12 +124,11 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* ...@@ -123,12 +124,11 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt*
if (cache == NULL) { if (cache == NULL) {
return 0; return 0;
} }
_cache_range_compare cmpFn = indexGetCompare(type);
MemTable* mem = cache; MemTable* mem = cache;
IndexCache* pCache = mem->pCache; IndexCache* pCache = mem->pCache;
_cache_range_compare cmpFn = indexGetCompare(type);
CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
pCt->colVal = term->colVal; pCt->colVal = term->colVal;
pCt->colType = term->colType; pCt->colType = term->colType;
...@@ -221,15 +221,18 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr ...@@ -221,15 +221,18 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
return cacheSearchCompareFunc_JSON(cache, term, tr, s, CONTAINS);
}
static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t cacheSearchEqual_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
return cacheSearchCompareFunc_JSON(cache, term, tr, s, EQ);
}
static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
return cacheSearchCompareFunc_JSON(cache, term, tr, s, CONTAINS);
}
static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
return cacheSearchCompareFunc_JSON(cache, term, tr, s, LT); return cacheSearchCompareFunc_JSON(cache, term, tr, s, LT);
} }
...@@ -267,13 +270,20 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR ...@@ -267,13 +270,20 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
int skip = 0; int skip = 0;
char* exBuf = NULL; char* exBuf = NULL;
if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { if (type == CONTAINS) {
SIndexTerm tm = {.suid = term->suid,
.operType = term->operType,
.colType = term->colType,
.colName = term->colVal,
.nColName = term->nColVal};
exBuf = indexPackJsonDataPrefixNoType(&tm, &skip);
pCt->colVal = exBuf;
} else {
exBuf = indexPackJsonDataPrefix(term, &skip); exBuf = indexPackJsonDataPrefix(term, &skip);
pCt->colVal = exBuf; pCt->colVal = exBuf;
} }
char* key = indexCacheTermGet(pCt); char* key = indexCacheTermGet(pCt);
// SSkipListIterator* iter = tSkipListCreateIter(mem->mem);
SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
while (tSkipListIterNext(iter)) { while (tSkipListIterNext(iter)) {
SSkipListNode* node = tSkipListIterGet(iter); SSkipListNode* node = tSkipListIterGet(iter);
...@@ -281,14 +291,22 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR ...@@ -281,14 +291,22 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
break; break;
} }
CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
// printf("json val: %s\n", c->colVal); TExeCond cond = CONTINUE;
if (0 != strncmp(c->colVal, pCt->colVal, skip)) { if (type == CONTAINS) {
break; if (0 == strncmp(c->colVal, pCt->colVal, skip)) {
cond = MATCH;
}
} else {
if (0 != strncmp(c->colVal, pCt->colVal, skip - 1)) {
break;
} else if (0 != strncmp(c->colVal, pCt->colVal, skip)) {
continue;
} else {
char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1);
memcpy(p, c->colVal, strlen(c->colVal));
cond = cmpFn(p + skip, term->colVal, dType);
}
} }
char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1);
memcpy(p, c->colVal, strlen(c->colVal));
TExeCond cond = cmpFn(p + skip, term->colVal, dType);
if (cond == MATCH) { if (cond == MATCH) {
if (c->operaType == ADD_VALUE) { if (c->operaType == ADD_VALUE) {
INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
...@@ -302,7 +320,6 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR ...@@ -302,7 +320,6 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
} else if (cond == BREAK) { } else if (cond == BREAK) {
break; break;
} }
taosMemoryFree(p);
} }
taosMemoryFree(pCt); taosMemoryFree(pCt);
......
source/libs/index/src/indexComm.c
浏览文件 @ e12576ed
...@@ -102,6 +102,10 @@ static TExeCond tCompareContains(void* a, void* b, int8_t type) { ...@@ -102,6 +102,10 @@ static TExeCond tCompareContains(void* a, void* b, int8_t type) {
__compar_fn_t func = indexGetCompar(type); __compar_fn_t func = indexGetCompar(type);
return tCompare(func, QUERY_TERM, a, b, type); return tCompare(func, QUERY_TERM, a, b, type);
} }
static TExeCond tCompareEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = indexGetCompar(type);
return tCompare(func, QUERY_TERM, a, b, type);
}
TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) { if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) {
return tDoCompare(func, cmptype, a, b); return tDoCompare(func, cmptype, a, b);
...@@ -186,9 +190,11 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { ...@@ -186,9 +190,11 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
} }
case QUERY_GREATER_EQUAL: { case QUERY_GREATER_EQUAL: {
if (ret >= 0) return MATCH; if (ret >= 0) return MATCH;
break;
} }
case QUERY_TERM: { case QUERY_TERM: {
if (ret == 0) return MATCH; if (ret == 0) return MATCH;
break;
} }
default: default:
return BREAK; return BREAK;
...@@ -197,7 +203,7 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { ...@@ -197,7 +203,7 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
} }
static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = { static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {
tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual, tCompareContains}; tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual, tCompareContains, tCompareEqual};
_cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; } _cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; }
...@@ -256,6 +262,26 @@ char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { ...@@ -256,6 +262,26 @@ char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
return buf; return buf;
} }
char* indexPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) {
/*
* |<-----colname---->|<-----dataType---->|<--------colVal---------->|
* |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
*/
uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType);
int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1;
char* buf = (char*)taosMemoryCalloc(1, sz);
char* p = buf;
memcpy(p, itm->colName, itm->nColName);
p += itm->nColName;
memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM));
p += sizeof(JSON_VALUE_DELIM);
*skip = p - buf;
return buf;
}
int32_t indexConvertData(void* src, int8_t type, void** dst) { int32_t indexConvertData(void* src, int8_t type, void** dst) {
int tlen = -1; int tlen = -1;
......
source/libs/index/src/indexFilter.c
浏览文件 @ e12576ed
...@@ -173,10 +173,8 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { ...@@ -173,10 +173,8 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
param->colId = l->colId; param->colId = l->colId;
param->colValType = l->node.resType.type; param->colValType = l->node.resType.type;
memcpy(param->dbName, l->dbName, sizeof(l->dbName)); memcpy(param->dbName, l->dbName, sizeof(l->dbName));
#pragma GCC diagnostic push memcpy(param->colName, r->literal, strlen(r->literal));
#pragma GCC diagnostic ignored "-Wformat-overflow" // sprintf(param->colName, "%s_%s", l->colName, r->literal);
sprintf(param->colName, "%s_%s", l->colName, r->literal);
#pragma GCC diagnostic pop
param->colValType = r->typeData; param->colValType = r->typeData;
return 0; return 0;
// memcpy(param->colName, l->colName, sizeof(l->colName)); // memcpy(param->colName, l->colName, sizeof(l->colName));
...@@ -188,6 +186,9 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { ...@@ -188,6 +186,9 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue)); SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
param->colId = -1; param->colId = -1;
param->colValType = (uint8_t)(vn->node.resType.type); param->colValType = (uint8_t)(vn->node.resType.type);
if (vn->literal == NULL || strlen(vn->literal) == 0) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
memcpy(param->colName, vn->literal, strlen(vn->literal)); memcpy(param->colName, vn->literal, strlen(vn->literal));
break; break;
} }
...@@ -340,9 +341,9 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { ...@@ -340,9 +341,9 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
return NULL; return NULL;
} }
static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
SIndexMetaArg *arg = &output->arg; int ret = 0;
int ret = 0;
SIndexMetaArg * arg = &output->arg;
EIndexQueryType qtype = 0; EIndexQueryType qtype = 0;
SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
if (left->colValType == TSDB_DATA_TYPE_JSON) { if (left->colValType == TSDB_DATA_TYPE_JSON) {
...@@ -506,7 +507,6 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { ...@@ -506,7 +507,6 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
if (nParam <= 1) { if (nParam <= 1) {
output->status = SFLT_NOT_INDEX; output->status = SFLT_NOT_INDEX;
return code; return code;
// SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
if (node->opType == OP_TYPE_JSON_GET_VALUE) { if (node->opType == OP_TYPE_JSON_GET_VALUE) {
return code; return code;
......
source/libs/index/src/indexJson.c
浏览文件 @ e12576ed
...@@ -22,6 +22,14 @@ int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { ...@@ -22,6 +22,14 @@ int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) {
int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
for (int i = 0; i < taosArrayGetSize(terms); i++) { for (int i = 0; i < taosArrayGetSize(terms); i++) {
SIndexJsonTerm *p = taosArrayGetP(terms, i); SIndexJsonTerm *p = taosArrayGetP(terms, i);
if (p->colType == TSDB_DATA_TYPE_BOOL) {
p->colType = TSDB_DATA_TYPE_INT;
} else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR ||
p->colType == TSDB_DATA_TYPE_BINARY) {
// p->colType = TSDB_DATA_TYPE_NCHAR;
} else {
p->colType = TSDB_DATA_TYPE_DOUBLE;
}
INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
} }
// handle put // handle put
...@@ -32,6 +40,14 @@ int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *re ...@@ -32,6 +40,14 @@ int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *re
SArray *terms = tq->query; SArray *terms = tq->query;
for (int i = 0; i < taosArrayGetSize(terms); i++) { for (int i = 0; i < taosArrayGetSize(terms); i++) {
SIndexJsonTerm *p = taosArrayGetP(terms, i); SIndexJsonTerm *p = taosArrayGetP(terms, i);
if (p->colType == TSDB_DATA_TYPE_BOOL) {
p->colType = TSDB_DATA_TYPE_INT;
} else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR ||
p->colType == TSDB_DATA_TYPE_BINARY) {
// p->colType = TSDB_DATA_TYPE_NCHAR;
} else {
p->colType = TSDB_DATA_TYPE_DOUBLE;
}
INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
} }
// handle search // handle search
......
source/libs/index/src/indexTfile.c
浏览文件 @ e12576ed
...@@ -73,6 +73,7 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTRslt* tr); ...@@ -73,6 +73,7 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, RangeType ctype); static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, RangeType ctype);
static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr); static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
static int32_t tfSearchEqual_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr); static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr); static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr); static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
...@@ -87,7 +88,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt ...@@ -87,7 +88,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
static int32_t (*tfSearch[][QUERY_MAX])(void* reader, SIndexTerm* tem, SIdxTRslt* tr) = { static int32_t (*tfSearch[][QUERY_MAX])(void* reader, SIndexTerm* tem, SIdxTRslt* tr) = {
{tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, tfSearchLessEqual, {tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, tfSearchLessEqual,
tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange},
{tfSearchTerm_JSON, tfSearchPrefix_JSON, tfSearchSuffix_JSON, tfSearchRegex_JSON, tfSearchLessThan_JSON, {tfSearchEqual_JSON, tfSearchPrefix_JSON, tfSearchSuffix_JSON, tfSearchRegex_JSON, tfSearchLessThan_JSON,
tfSearchLessEqual_JSON, tfSearchGreaterThan_JSON, tfSearchGreaterEqual_JSON, tfSearchRange_JSON}}; tfSearchLessEqual_JSON, tfSearchGreaterThan_JSON, tfSearchGreaterEqual_JSON, tfSearchRange_JSON}};
TFileCache* tfileCacheCreate(const char* path) { TFileCache* tfileCacheCreate(const char* path) {
...@@ -424,6 +425,9 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { ...@@ -424,6 +425,9 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
// deprecate api // deprecate api
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t tfSearchEqual_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
return tfSearchCompareFunc_JSON(reader, tem, tr, EQ);
}
static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
return tfSearchCompareFunc_JSON(reader, tem, tr, CONTAINS); return tfSearchCompareFunc_JSON(reader, tem, tr, CONTAINS);
} }
...@@ -456,7 +460,17 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt ...@@ -456,7 +460,17 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
int ret = 0; int ret = 0;
int skip = 0; int skip = 0;
char* p = indexPackJsonDataPrefix(tem, &skip); char* p = NULL;
if (ctype == CONTAINS) {
SIndexTerm tm = {.suid = tem->suid,
.operType = tem->operType,
.colType = tem->colType,
.colName = tem->colVal,
.nColName = tem->nColVal};
p = indexPackJsonDataPrefixNoType(&tm, &skip);
} else {
p = indexPackJsonDataPrefix(tem, &skip);
}
_cache_range_compare cmpFn = indexGetCompare(ctype); _cache_range_compare cmpFn = indexGetCompare(ctype);
...@@ -470,18 +484,22 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt ...@@ -470,18 +484,22 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
FstSlice* s = &rt->data; FstSlice* s = &rt->data;
int32_t sz = 0; int32_t sz = 0;
char* ch = (char*)fstSliceData(s, &sz); char* ch = (char*)fstSliceData(s, &sz);
char* tmp = taosMemoryCalloc(1, sz + 1); TExeCond cond = CONTINUE;
memcpy(tmp, ch, sz); if (ctype == CONTAINS) {
if (0 == strncmp(ch, p, skip)) {
if (0 != strncmp(tmp, p, skip)) { cond = MATCH;
swsResultDestroy(rt); }
taosMemoryFree(tmp); } else {
break; if (0 != strncmp(ch, p, skip - 1)) {
swsResultDestroy(rt);
break;
} else if (0 != strncmp(ch, p, skip)) {
continue;
}
cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
} }
TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
if (MATCH == cond) { if (MATCH == cond) {
tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
} else if (CONTINUE == cond) { } else if (CONTINUE == cond) {
...@@ -489,7 +507,6 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt ...@@ -489,7 +507,6 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
swsResultDestroy(rt); swsResultDestroy(rt);
break; break;
} }
taosMemoryFree(tmp);
swsResultDestroy(rt); swsResultDestroy(rt);
} }
streamWithStateDestroy(st); streamWithStateDestroy(st);
......
source/libs/index/test/jsonUT.cc
浏览文件 @ e12576ed
...@@ -107,41 +107,40 @@ TEST_F(JsonEnv, testWrite) { ...@@ -107,41 +107,40 @@ TEST_F(JsonEnv, testWrite) {
{ {
std::string colName("test"); std::string colName("test");
std::string colVal("ab"); std::string colVal("ab");
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), for (int i = 0; i < 100; i++) {
colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate(); SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term); indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("voltage"); std::string colName("voltage");
std::string colVal("ab1"); std::string colVal("ab1");
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), for (int i = 0; i < 100; i++) {
colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate(); SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term); indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("voltage"); std::string colName("voltage");
std::string colVal("123"); std::string colVal("123");
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) { for (size_t i = 0; i < 100; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test"); std::string colName("test");
...@@ -154,7 +153,7 @@ TEST_F(JsonEnv, testWrite) { ...@@ -154,7 +153,7 @@ TEST_F(JsonEnv, testWrite) {
SArray* result = taosArrayInit(1, sizeof(uint64_t)); SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_TERM); indexMultiTermQueryAdd(mq, q, QUERY_TERM);
tIndexJsonSearch(index, mq, result); tIndexJsonSearch(index, mq, result);
assert(100 == taosArrayGetSize(result)); EXPECT_EQ(100, taosArrayGetSize(result));
indexMultiTermQueryDestroy(mq); indexMultiTermQueryDestroy(mq);
} }
} }
...@@ -162,44 +161,44 @@ TEST_F(JsonEnv, testWriteMillonData) { ...@@ -162,44 +161,44 @@ TEST_F(JsonEnv, testWriteMillonData) {
{ {
std::string colName("test"); std::string colName("test");
std::string colVal("ab"); std::string colVal("ab");
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 10; i++) { for (size_t i = 0; i < 10; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("voltagefdadfa"); std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx"); std::string colVal("abxxxxxxxxxxxx");
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
colVal[i % colVal.size()] = '0' + i % 128; colVal[i % colVal.size()] = '0' + i % 128;
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) { for (size_t i = 0; i < 100; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
} }
{ {
std::string colName("voltagefdadfa"); std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx"); std::string colVal("abxxxxxxxxxxxx");
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test"); std::string colName("test");
...@@ -227,7 +226,7 @@ TEST_F(JsonEnv, testWriteMillonData) { ...@@ -227,7 +226,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
SArray* result = taosArrayInit(1, sizeof(uint64_t)); SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
tIndexJsonSearch(index, mq, result); tIndexJsonSearch(index, mq, result);
assert(0 == taosArrayGetSize(result)); EXPECT_EQ(0, taosArrayGetSize(result));
indexMultiTermQueryDestroy(mq); indexMultiTermQueryDestroy(mq);
} }
{ {
...@@ -252,55 +251,55 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { ...@@ -252,55 +251,55 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
{ {
std::string colName("test"); std::string colName("test");
// std::string colVal("10"); // std::string colVal("10");
int val = 10; int val = 10;
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test2"); std::string colName("test2");
int val = 20; int val = 20;
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test"); std::string colName("test");
int val = 15; int val = 15;
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test2"); std::string colName("test2");
const char* val = "test"; const char* val = "test";
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
(const char*)val, strlen(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
(const char*)val, strlen(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test"); std::string colName("test");
...@@ -380,28 +379,28 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { ...@@ -380,28 +379,28 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
{ {
std::string colName("test1"); std::string colName("test1");
int val = 10; int val = 10;
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test"); std::string colName("test");
std::string colVal("xxxxxxxxxxxxxxxxxxx"); std::string colVal("xxxxxxxxxxxxxxxxxxx");
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test1"); std::string colName("test1");
...@@ -478,15 +477,15 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { ...@@ -478,15 +477,15 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
std::string colName("other_column"); std::string colName("other_column");
int val = 100; int val = 100;
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i); tIndexJsonPut(index, terms, i);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test1"); std::string colName("test1");
...@@ -506,15 +505,15 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { ...@@ -506,15 +505,15 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
{ {
std::string colName("test1"); std::string colName("test1");
int val = 15; int val = 15;
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) { for (size_t i = 0; i < 1000; i++) {
SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, i + 1000); tIndexJsonPut(index, terms, i + 1000);
indexMultiTermDestroy(terms);
} }
indexMultiTermDestroy(terms);
} }
{ {
std::string colName("test1"); std::string colName("test1");
......
source/libs/scalar/src/sclvector.c
浏览文件 @ e12576ed
...@@ -633,7 +633,8 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { ...@@ -633,7 +633,8 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
if (IS_VAR_DATA_TYPE(inType)) { if (IS_VAR_DATA_TYPE(inType)) {
return vectorConvertFromVarData(pIn, pOut, inType, outType); return vectorConvertFromVarData(pIn, pOut, inType, outType);
} }
pOut->numOfRows = pIn->numOfRows;
switch (outType) { switch (outType) {
case TSDB_DATA_TYPE_BOOL: { case TSDB_DATA_TYPE_BOOL: {
for (int32_t i = 0; i < pIn->numOfRows; ++i) { for (int32_t i = 0; i < pIn->numOfRows; ++i) {
......
source/libs/transport/src/transCli.c
浏览文件 @ e12576ed
...@@ -338,8 +338,7 @@ void cliHandleResp(SCliConn* conn) { ...@@ -338,8 +338,7 @@ void cliHandleResp(SCliConn* conn) {
return; return;
} }
int ret = cliAppCb(conn, &transMsg, pMsg); if (cliAppCb(conn, &transMsg, pMsg) != 0) {
if (ret != 0) {
tTrace("try to send req to next node"); tTrace("try to send req to next node");
return; return;
} }
...@@ -403,15 +402,13 @@ void cliHandleExcept(SCliConn* pConn) { ...@@ -403,15 +402,13 @@ void cliHandleExcept(SCliConn* pConn) {
continue; continue;
} }
} }
int ret = cliAppCb(pConn, &transMsg, pMsg); if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
if (ret != 0) {
tTrace("try to send req to next node"); tTrace("try to send req to next node");
return; return;
} }
destroyCmsg(pMsg); destroyCmsg(pMsg);
tTrace("%s cli conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); tTrace("%s cli conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn);
} while (!transQueueEmpty(&pConn->cliMsgs)); } while (!transQueueEmpty(&pConn->cliMsgs));
transUnrefCliHandle(pConn); transUnrefCliHandle(pConn);
} }
...@@ -976,7 +973,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { ...@@ -976,7 +973,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
arg->param1 = pMsg; arg->param1 = pMsg;
arg->param2 = pThrd; arg->param2 = pThrd;
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
cliDestroyConn(pConn, true); transUnrefCliHandle(pConn);
return -1; return -1;
} }
} else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {
......
source/util/src/talgo.c
浏览文件 @ e12576ed
...@@ -158,82 +158,48 @@ void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ ...@@ -158,82 +158,48 @@ void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
} }
void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t compar, int32_t flags) { void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size, __compar_fn_t compar, int32_t flags) {
// TODO: need to check the correctness of this function uint8_t *p;
int32_t l = 0; int32_t lidx;
int32_t r = (int32_t)nmemb; int32_t ridx;
int32_t idx = 0; int32_t midx;
int32_t comparison; int32_t c;
if (nmemb <= 0) return NULL;
lidx = 0;
ridx = nmemb - 1;
while (lidx <= ridx) {
midx = (lidx + ridx) / 2;
p = (uint8_t *)base + size * midx;
c = compar(key, p);
if (c == 0) {
break;
} else if (c < 0) {
ridx = midx - 1;
} else {
lidx = midx + 1;
}
}
if (flags == TD_EQ) { if (flags == TD_EQ) {
return bsearch(key, base, nmemb, size, compar); return c ? NULL : p;
} else if (flags == TD_GE) { } else if (flags == TD_GE) {
if (nmemb <= 0) return NULL; return (c <= 0) ? p : (midx + 1 < nmemb ? p + size : NULL);
if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0);
if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL;
while (l < r) {
idx = (l + r) / 2;
comparison = (*compar)(key, elePtrAt(base, size, idx));
if (comparison < 0) {
r = idx;
} else if (comparison > 0) {
l = idx + 1;
} else {
return elePtrAt(base, size, idx);
}
}
if ((*compar)(key, elePtrAt(base, size, idx)) < 0) {
return elePtrAt(base, size, idx);
} else {
if (idx + 1 > nmemb - 1) {
return NULL;
} else {
return elePtrAt(base, size, idx + 1);
}
}
} else if (flags == TD_LE) { } else if (flags == TD_LE) {
if (nmemb <= 0) return NULL; return (c >= 0) ? p : (midx > 0 ? p - size : NULL);
if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1);
if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL;
while (l < r) {
idx = (l + r) / 2;
comparison = (*compar)(key, elePtrAt(base, size, idx));
if (comparison < 0) {
r = idx;
} else if (comparison > 0) {
l = idx + 1;
} else {
return elePtrAt(base, size, idx);
}
}
if ((*compar)(key, elePtrAt(base, size, idx)) > 0) {
return elePtrAt(base, size, idx);
} else {
if (idx == 0) {
return NULL;
} else {
return elePtrAt(base, size, idx - 1);
}
}
} else { } else {
assert(0); ASSERT(0);
return NULL;
} }
return NULL;
} }
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char* buf, bool maxroot) { __ext_compar_fn_t compar, char *buf, bool maxroot) {
int32_t parent; int32_t parent;
int32_t child; int32_t child;
char* tmp = NULL; char *tmp = NULL;
if (buf == NULL) { if (buf == NULL) {
tmp = taosMemoryMalloc(size); tmp = taosMemoryMalloc(size);
} else { } else {
...@@ -288,7 +254,7 @@ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, ...@@ -288,7 +254,7 @@ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar,
bool maxroot) { bool maxroot) {
int32_t i; int32_t i;
char* buf = taosMemoryCalloc(1, size); char *buf = taosMemoryCalloc(1, size);
if (buf == NULL) { if (buf == NULL) {
return; return;
} }
......
source/util/test/CMakeLists.txt
浏览文件 @ e12576ed
...@@ -67,4 +67,12 @@ target_link_libraries(bloomFilterTest os util gtest_main) ...@@ -67,4 +67,12 @@ target_link_libraries(bloomFilterTest os util gtest_main)
add_test( add_test(
NAME bloomFilterTest NAME bloomFilterTest
COMMAND bloomFilterTest COMMAND bloomFilterTest
)
# taosbsearchTest
add_executable(taosbsearchTest "taosbsearchTest.cpp")
target_link_libraries(taosbsearchTest os util gtest_main)
add_test(
NAME taosbsearchTest
COMMAND taosbsearchTest
) )
\ No newline at end of file
tests/script/tsim/query/explain.sim
浏览文件 @ e12576ed
...@@ -42,14 +42,12 @@ sql explain select count(*),sum(f1) from tb1; ...@@ -42,14 +42,12 @@ sql explain select count(*),sum(f1) from tb1;
sql explain select count(*),sum(f1) from st1; sql explain select count(*),sum(f1) from st1;
sql explain select count(*),sum(f1) from st1 group by f1; sql explain select count(*),sum(f1) from st1 group by f1;
#sql explain select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); #sql explain select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev);
sql explain select min(f1) from st1 interval(1m, 2a) sliding(30s);
print ======== step3 print ======== step3
sql explain verbose true select * from st1 where -2; sql explain verbose true select * from st1 where -2;
sql explain verbose true select ts from tb1 where f1 > 0; sql explain verbose true select ts from tb1 where f1 > 0;
sql explain verbose true select * from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00'; sql explain verbose true select * from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00';
sql explain verbose true select * from information_schema.user_stables where db_name='db2'; sql explain verbose true select * from information_schema.user_stables where db_name='db2';
sql explain verbose true select count(*),sum(f1) from st1 where f1 > 0 and ts > '2021-10-31 00:00:00' group by f1 having sum(f1) > 0;
print ======== step4 print ======== step4
sql explain analyze select ts from st1 where -2; sql explain analyze select ts from st1 where -2;
...@@ -61,8 +59,6 @@ sql explain analyze select * from information_schema.user_stables; ...@@ -61,8 +59,6 @@ sql explain analyze select * from information_schema.user_stables;
sql explain analyze select count(*),sum(f1) from tb1; sql explain analyze select count(*),sum(f1) from tb1;
sql explain analyze select count(*),sum(f1) from st1; sql explain analyze select count(*),sum(f1) from st1;
sql explain analyze select count(*),sum(f1) from st1 group by f1; sql explain analyze select count(*),sum(f1) from st1 group by f1;
#sql explain analyze select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev);
sql explain analyze select min(f1) from st1 interval(3m, 2a) sliding(1m);
print ======== step5 print ======== step5
sql explain analyze verbose true select ts from st1 where -2; sql explain analyze verbose true select ts from st1 where -2;
...@@ -78,8 +74,6 @@ sql explain analyze verbose true select count(*),sum(f1) from st1 group by f1; ...@@ -78,8 +74,6 @@ sql explain analyze verbose true select count(*),sum(f1) from st1 group by f1;
sql explain analyze verbose true select ts from tb1 where f1 > 0; sql explain analyze verbose true select ts from tb1 where f1 > 0;
sql explain analyze verbose true select f1 from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00'; sql explain analyze verbose true select f1 from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00';
sql explain analyze verbose true select * from information_schema.user_stables where db_name='db2'; sql explain analyze verbose true select * from information_schema.user_stables where db_name='db2';
sql explain analyze verbose true select count(*),sum(f1) from st1 where f1 > 0 and ts > '2021-10-31 00:00:00' group by f1 having sum(f1) > 0;
sql explain analyze verbose true select min(f1) from st1 interval(3m, 2a) sliding(1m);
sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0; sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0;
#not pass case #not pass case
...@@ -93,6 +87,12 @@ sql explain analyze verbose true select * from (select min(f1),count(*) a from s ...@@ -93,6 +87,12 @@ sql explain analyze verbose true select * from (select min(f1),count(*) a from s
#sql explain select * from tb1, tb2 where tb1.ts=tb2.ts; #sql explain select * from tb1, tb2 where tb1.ts=tb2.ts;
#sql explain select * from st1, st2 where tb1.ts=tb2.ts; #sql explain select * from st1, st2 where tb1.ts=tb2.ts;
#sql explain analyze verbose true select sum(a+b) from (select _rowts, min(f1) b,count(*) a from st1 where f1 > 0 interval(1a)) where a < 0 interval(1s); #sql explain analyze verbose true select sum(a+b) from (select _rowts, min(f1) b,count(*) a from st1 where f1 > 0 interval(1a)) where a < 0 interval(1s);
#sql explain select min(f1) from st1 interval(1m, 2a) sliding(30s);
#sql explain verbose true select count(*),sum(f1) from st1 where f1 > 0 and ts > '2021-10-31 00:00:00' group by f1 having sum(f1) > 0;
#sql explain analyze select min(f1) from st1 interval(3m, 2a) sliding(1m);
#sql explain analyze select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev);
#sql explain analyze verbose true select count(*),sum(f1) from st1 where f1 > 0 and ts > '2021-10-31 00:00:00' group by f1 having sum(f1) > 0;
#sql explain analyze verbose true select min(f1) from st1 interval(3m, 2a) sliding(1m);
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
tests/system-test/7-tmq/db.py
浏览文件 @ e12576ed
...@@ -429,12 +429,6 @@ class TDTestCase: ...@@ -429,12 +429,6 @@ class TDTestCase:
tdLog.printNoPrefix("======== test case 2 end ...... ") tdLog.printNoPrefix("======== test case 2 end ...... ")
def tmqCase3(self, cfgPath, buildPath):
tdLog.printNoPrefix("======== test case 3: ")
tdLog.printNoPrefix("======== test case 3 end ...... ")
def run(self): def run(self):
tdSql.prepare() tdSql.prepare()
...@@ -446,11 +440,8 @@ class TDTestCase: ...@@ -446,11 +440,8 @@ class TDTestCase:
cfgPath = buildPath + "/../sim/psim/cfg" cfgPath = buildPath + "/../sim/psim/cfg"
tdLog.info("cfgPath: %s" % cfgPath) tdLog.info("cfgPath: %s" % cfgPath)
# self.tmqCase1(cfgPath, buildPath) self.tmqCase1(cfgPath, buildPath)
self.tmqCase2(cfgPath, buildPath) self.tmqCase2(cfgPath, buildPath)
# self.tmqCase3(cfgPath, buildPath)
# self.tmqCase4(cfgPath, buildPath)
# self.tmqCase5(cfgPath, buildPath)
def stop(self): def stop(self):
tdSql.close() tdSql.close()
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或