Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 312b7a4f 编写于 作者: J jiacy-jcy
浏览文件
操作

Merge branch '3.0' into test/jcy

上级 0c8c9552 db2b9931
展开全部 显示空白变更内容
内联 并排
Showing with 1917 addition and 1159 deletion
Jenkinsfile2
浏览文件 @ 312b7a4f
......@@ -88,12 +88,6 @@ def pre_test(){
cmake .. > /dev/null
make -j4> /dev/null
'''
sh'''
cd ${WKPY}
git reset --hard
git pull
pip3 install .
'''
return 1
}
......@@ -103,7 +97,6 @@ pipeline {
environment{
WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDengine'
WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
}
stages {
stage('pre_build'){
......@@ -124,11 +117,6 @@ pipeline {
./test-all.sh b1fq
'''
sh'''
export LD_LIBRARY_PATH=${WKC}/debug/build/lib
cd ${WKC}/tests/system-test
./fulltest.sh
'''
sh'''
cd ${WKC}/debug
ctest
'''
......
include/client/taos.h
浏览文件 @ 312b7a4f
......@@ -146,6 +146,7 @@ DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt);
DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
......
include/common/taosdef.h
浏览文件 @ 312b7a4f
......@@ -78,13 +78,6 @@ typedef enum {
TSDB_SMA_TYPE_ROLLUP = 2, // Rollup SMA
} ETsdbSmaType;
typedef enum {
TSDB_BSMA_TYPE_NONE = 0, // no block-wise SMA
TSDB_BSMA_TYPE_I = 1, // sum/min/max(default)
} ETsdbBSmaType;
#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I
extern char *qtypeStr[];
#define TSDB_PORT_HTTP 11
......
include/common/tmsg.h
浏览文件 @ 312b7a4f
......@@ -326,6 +326,13 @@ int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp);
int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
void* taosDecodeSEpSet(const void* buf, SEpSet* pEp);
typedef struct {
SEpSet epSet;
} SMEpSet;
int32_t tSerializeSMEpSet(void* buf, int32_t bufLen, SMEpSet* pReq);
int32_t tDeserializeSMEpSet(void* buf, int32_t buflen, SMEpSet* pReq);
typedef struct {
int8_t connType;
int32_t pid;
......@@ -682,6 +689,7 @@ int32_t tDeserializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq);
typedef struct {
int32_t numOfFuncs;
bool ignoreCodeComment;
SArray* pFuncNames;
} SRetrieveFuncReq;
......@@ -710,6 +718,7 @@ typedef struct {
int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp);
int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp);
void tFreeSFuncInfo(SFuncInfo *pInfo);
void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp);
typedef struct {
......@@ -982,7 +991,6 @@ int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
void tFreeSShowRsp(SShowRsp* pRsp);
typedef struct {
int32_t type;
char db[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN];
int64_t showId;
......@@ -1478,8 +1486,12 @@ typedef struct {
typedef struct {
float xFilesFactor;
int32_t delay;
int8_t nFuncIds;
int32_t qmsg1Len;
int32_t qmsg2Len;
func_id_t* pFuncIds;
char* qmsg1; // not null: pAst1:qmsg1:SRetention1 => trigger aggr task1
char* qmsg2; // not null: pAst2:qmsg2:SRetention2 => trigger aggr task2
int8_t nFuncIds;
} SRSmaParam;
typedef struct SVCreateTbReq {
......@@ -2329,9 +2341,10 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) {
}
}
static FORCE_INLINE void tdFreeTSmaWrapper(STSmaWrapper* pSW) {
static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) {
tdDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW);
taosMemoryFree(pSW);
return NULL;
}
static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
......@@ -2719,6 +2732,7 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
}
return buf;
}
#pragma pack(pop)
#ifdef __cplusplus
......
include/libs/catalog/catalog.h
浏览文件 @ 312b7a4f
......@@ -224,6 +224,8 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo);
/**
* Destroy catalog and relase all resources
......
include/libs/function/function.h
浏览文件 @ 312b7a4f
......@@ -166,6 +166,7 @@ typedef struct SInputColumnInfoData {
SColumnInfoData *pPTS; // primary timestamp column
SColumnInfoData **pData;
SColumnDataAgg **pColumnDataAgg;
uint64_t uid; // table uid
} SInputColumnInfoData;
// sql function runtime context
......@@ -191,7 +192,7 @@ typedef struct SqlFunctionCtx {
int16_t functionId; // function id
char * pOutput; // final result output buffer, point to sdata->data
int32_t numOfParams;
SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
int64_t *ptsList; // corresponding timestamp array list
SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
int32_t offset;
......
include/libs/function/functionMgt.h
浏览文件 @ 312b7a4f
......@@ -119,12 +119,19 @@ typedef enum EFunctionType {
struct SqlFunctionCtx;
struct SResultRowEntryInfo;
struct STimeWindow;
struct SCatalog;
typedef struct SFmGetFuncInfoParam {
struct SCatalog* pCtg;
void *pRpc;
const SEpSet* pMgmtEps;
} SFmGetFuncInfoParam;
int32_t fmFuncMgtInit();
void fmFuncMgtDestroy();
int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
......
include/libs/transport/trpc.h
浏览文件 @ 312b7a4f
......@@ -54,12 +54,13 @@ typedef struct {
uint16_t clientPort;
SRpcMsg rpcMsg;
int32_t rspLen;
void *pRsp;
void *pNode;
void * pRsp;
void * pNode;
} SNodeMsg;
typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
typedef struct SRpcInit {
uint16_t localPort; // local port
......@@ -80,7 +81,10 @@ typedef struct SRpcInit {
RpcCfp cfp;
// call back to retrieve the client auth info, for server app only
RpcAfp afp;;
RpcAfp afp;
// user defined retry func
RpcRfp rfp;
void *parent;
} SRpcInit;
......@@ -93,7 +97,7 @@ typedef struct {
typedef struct {
int32_t msgType;
void *val;
void * val;
int32_t (*clone)(void *src, void **dst);
void (*freeFunc)(const void *arg);
} SRpcBrokenlinkVal;
......
include/os/osEnv.h
浏览文件 @ 312b7a4f
......@@ -34,6 +34,7 @@ extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern float tsNumOfCores;
extern int64_t tsTotalMemoryKB;
extern char* tsProcPath;
extern char configDir[];
extern char tsDataDir[];
......
include/util/talgo.h
浏览文件 @ 312b7a4f
......@@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
* @return
*/
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
__ext_compar_fn_t compar, char* buf, bool maxroot);
/**
* sort heap to make sure it is a max/min root heap
......@@ -98,7 +98,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
* @return
*/
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
const void *parswap, __ext_swap_fn_t swap, bool maxroot);
bool maxroot);
#ifdef __cplusplus
}
......
source/client/inc/clientInt.h
浏览文件 @ 312b7a4f
......@@ -219,6 +219,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo);
void setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
void doFreeReqResultInfo(SReqResultInfo* pResInfo);
static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
SMqRspObj* msg = (SMqRspObj*)res;
......
source/client/inc/clientStmt.h
浏览文件 @ 312b7a4f
......@@ -60,6 +60,7 @@ typedef struct SStmtBindInfo {
} SStmtBindInfo;
typedef struct SStmtExecInfo {
int32_t affectedRows;
SRequestObj* pRequest;
SHashObj* pVgHash;
SHashObj* pBlockHash;
......
source/client/src/clientEnv.c
浏览文件 @ 312b7a4f
......@@ -49,8 +49,8 @@ static void registerRequest(SRequestObj *pRequest) {
if (pTscObj->pAppInfo) {
SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary;
int32_t total = atomic_add_fetch_64((int64_t*)&pSummary->totalRequests, 1);
int32_t currentInst = atomic_add_fetch_64((int64_t*)&pSummary->currentRequests, 1);
int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1);
int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
......@@ -60,16 +60,16 @@ static void registerRequest(SRequestObj *pRequest) {
static void deregisterRequest(SRequestObj *pRequest) {
assert(pRequest != NULL);
STscObj * pTscObj = pRequest->pTscObj;
STscObj *pTscObj = pRequest->pTscObj;
SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
int32_t currentInst = atomic_sub_fetch_64((int64_t*)&pActivity->currentRequests, 1);
int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
" ms, current:%d, app current:%d",
pRequest->self, pTscObj->id, pRequest->requestId, duration/1000, num, currentInst);
pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
releaseTscObj(pTscObj->id);
}
......@@ -160,13 +160,9 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI
return pObj;
}
STscObj *acquireTscObj(int64_t rid) {
return (STscObj *)taosAcquireRef(clientConnRefPool, rid);
}
STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientConnRefPool, rid); }
int32_t releaseTscObj(int64_t rid) {
return taosReleaseRef(clientConnRefPool, rid);
}
int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); }
void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) {
assert(pObj != NULL);
......@@ -194,7 +190,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
return pRequest;
}
static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
taosMemoryFreeClear(pResInfo->pRspMsg);
taosMemoryFreeClear(pResInfo->length);
taosMemoryFreeClear(pResInfo->row);
......@@ -244,14 +240,9 @@ void destroyRequest(SRequestObj *pRequest) {
taosRemoveRef(clientReqRefPool, pRequest->self);
}
SRequestObj *acquireRequest(int64_t rid) {
return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid);
}
int32_t releaseRequest(int64_t rid) {
return taosReleaseRef(clientReqRefPool, rid);
}
SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); }
int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); }
void taos_init_imp(void) {
// In the APIs of other program language, taos_cleanup is not available yet.
......
source/client/src/clientMain.c
浏览文件 @ 312b7a4f
......@@ -135,6 +135,16 @@ void taos_free_result(TAOS_RES *res) {
if (TD_RES_QUERY(res)) {
SRequestObj *pRequest = (SRequestObj *)res;
destroyRequest(pRequest);
} else if (TD_RES_TMQ(res)) {
SMqRspObj *pRsp = (SMqRspObj *)res;
if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema);
if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName);
if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags);
if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema);
pRsp->resInfo.pRspMsg = NULL;
doFreeReqResultInfo(&pRsp->resInfo);
}
}
......@@ -622,6 +632,10 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
return stmtSetTbName(stmt, name);
}
int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) {
return taos_stmt_set_tbname(stmt, name);
}
int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
if (stmt == NULL || bind == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
......
source/client/src/clientStmt.c
浏览文件 @ 312b7a4f
......@@ -486,7 +486,8 @@ int stmtExec(TAOS_STMT *stmt) {
STMT_ERR_JRET(pStmt->exec.pRequest->code);
pStmt->affectedRows += taos_affected_rows(pStmt->exec.pRequest);
pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest);
pStmt->affectedRows += pStmt->exec.affectedRows;
_return:
......
source/client/test/clientTests.cpp
浏览文件 @ 312b7a4f
......@@ -662,7 +662,7 @@ TEST(testCase, agg_query_tables) {
TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "select count(*) from tu");
pRes = taos_query(pConn, "select now() from m1");
if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
......
source/common/src/tdataformat.c
浏览文件 @ 312b7a4f
......@@ -110,7 +110,7 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
for (int i = 0; i < numOfCols; i++) {
col_type_t type = 0;
int8_t sma = TSDB_BSMA_TYPE_NONE;
int8_t sma = 0;
col_id_t colId = 0;
col_bytes_t bytes = 0;
buf = taosDecodeFixedI8(buf, &type);
......
source/common/src/tmsg.c
浏览文件 @ 312b7a4f
......@@ -434,6 +434,15 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
for (int8_t i = 0; i < param->nFuncIds; ++i) {
tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
}
tlen += taosEncodeFixedI32(buf, param->qmsg1Len);
if (param->qmsg1Len > 0) {
tlen += taosEncodeString(buf, param->qmsg1);
}
tlen += taosEncodeFixedI32(buf, param->qmsg2Len);
if (param->qmsg2Len > 0) {
tlen += taosEncodeString(buf, param->qmsg2);
}
}
break;
case TD_CHILD_TABLE:
......@@ -496,18 +505,25 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
}
if (pReq->rollup) {
pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
buf = taosDecodeFixedI32(buf, &param->delay);
buf = taosDecodeFixedI8(buf, &param->nFuncIds);
if (param->nFuncIds > 0) {
param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
param->pFuncIds = (func_id_t *)taosMemoryCalloc(param->nFuncIds, sizeof(func_id_t));
for (int8_t i = 0; i < param->nFuncIds; ++i) {
buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
}
} else {
param->pFuncIds = NULL;
}
buf = taosDecodeFixedI32(buf, &param->qmsg1Len);
if (param->qmsg1Len > 0) {
buf = taosDecodeString(buf, &param->qmsg1);
}
buf = taosDecodeFixedI32(buf, &param->qmsg2Len);
if (param->qmsg2Len > 0) {
buf = taosDecodeString(buf, &param->qmsg2);
}
} else {
pReq->stbCfg.pRSmaParam = NULL;
......@@ -828,6 +844,27 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) {
taosArrayDestroy(pReq->pFields);
pReq->pFields = NULL;
}
int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) return -1;
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
}
int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) {
SCoder encoder = {0};
......@@ -1624,6 +1661,7 @@ int32_t tSerializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq *
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1;
if (tEncodeI8(&encoder, pReq->ignoreCodeComment) < 0) return -1;
if (pReq->numOfFuncs != (int32_t)taosArrayGetSize(pReq->pFuncNames)) return -1;
for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
......@@ -1644,6 +1682,7 @@ int32_t tDeserializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1;
if (tDecodeI8(&decoder, (int8_t *)&pReq->ignoreCodeComment) < 0) return -1;
pReq->pFuncNames = taosArrayInit(pReq->numOfFuncs, TSDB_FUNC_NAME_LEN);
if (pReq->pFuncNames == NULL) return -1;
......@@ -1681,9 +1720,13 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *
if (tEncodeI64(&encoder, pInfo->signature) < 0) return -1;
if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1;
if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1;
if (pInfo->codeSize) {
if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1;
}
if (pInfo->commentSize) {
if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1;
}
}
tEndEncode(&encoder);
......@@ -1713,15 +1756,23 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp
if (tDecodeI64(&decoder, &fInfo.signature) < 0) return -1;
if (tDecodeI32(&decoder, &fInfo.codeSize) < 0) return -1;
if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1;
if (fInfo.codeSize) {
fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize);
fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize);
if (fInfo.pCode == NULL || fInfo.pComment == NULL) {
if (fInfo.pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1;
}
if (fInfo.commentSize) {
fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize);
if (fInfo.pComment == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1;
}
taosArrayPush(pRsp->pFuncInfos, &fInfo);
}
tEndDecode(&decoder);
......@@ -1730,12 +1781,20 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp
return 0;
}
void tFreeSFuncInfo(SFuncInfo *pInfo) {
if (NULL == pInfo) {
return;
}
taosMemoryFree(pInfo->pCode);
taosMemoryFree(pInfo->pComment);
}
void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) {
int32_t size = taosArrayGetSize(pRsp->pFuncInfos);
for (int32_t i = 0; i < size; ++i) {
SFuncInfo *pInfo = taosArrayGet(pRsp->pFuncInfos, i);
taosMemoryFree(pInfo->pCode);
taosMemoryFree(pInfo->pComment);
tFreeSFuncInfo(pInfo);
}
taosArrayDestroy(pRsp->pFuncInfos);
}
......@@ -2403,7 +2462,6 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI64(&encoder, pReq->showId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->type) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
tEndEncode(&encoder);
......@@ -2419,7 +2477,6 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
tEndDecode(&decoder);
......
source/dnode/mgmt/implement/src/dmHandle.c
浏览文件 @ 312b7a4f
......@@ -216,6 +216,126 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) {
dmStopStatusThread(pWrapper->pDnode);
}
static int32_t dmSpawnUdfd(SDnode *pDnode);
void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) {
dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
uv_close((uv_handle_t*)process, NULL);
SDnode *pDnode = process->data;
SUdfdData *pData = &pDnode->udfdData;
if (atomic_load_8(&pData->stopping) != 0) {
dDebug("udfd process exit due to stopping");
} else {
uv_close((uv_handle_t*)&pData->ctrlPipe, NULL);
dmSpawnUdfd(pDnode);
}
}
static int32_t dmSpawnUdfd(SDnode *pDnode) {
dInfo("dnode start spawning udfd");
uv_process_options_t options = {0};
char path[PATH_MAX] = {0};
if (tsProcPath == NULL) {
path[0] = '.';
} else {
strncpy(path, tsProcPath, strlen(tsProcPath));
taosDirName(path);
}
strcat(path, "/udfd");
char* argsUdfd[] = {path, "-c", configDir, NULL};
options.args = argsUdfd;
options.file = path;
options.exit_cb = dmUdfdExit;
SUdfdData *pData = &pDnode->udfdData;
uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1);
uv_stdio_container_t child_stdio[3];
child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE;
child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe;
child_stdio[1].flags = UV_IGNORE;
child_stdio[2].flags = UV_INHERIT_FD;
child_stdio[2].data.fd = 2;
options.stdio_count = 3;
options.stdio = child_stdio;
char dnodeIdEnvItem[32] = {0};
char thrdPoolSizeEnvItem[32] = {0};
snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId);
float numCpuCores = 4;
taosGetCpuCores(&numCpuCores);
snprintf(thrdPoolSizeEnvItem,32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2);
char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL};
options.env = envUdfd;
int err = uv_spawn(&pData->loop, &pData->process, &options);
pData->process.data = (void*)pDnode;
if (err != 0) {
dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err));
}
return err;
}
static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) {
if (!uv_is_closing(handle)) {
uv_close(handle, NULL);
}
}
void dmWatchUdfd(void *args) {
SDnode *pDnode = args;
SUdfdData *pData = &pDnode->udfdData;
uv_loop_init(&pData->loop);
int32_t err = dmSpawnUdfd(pDnode);
atomic_store_32(&pData->spawnErr, err);
uv_barrier_wait(&pData->barrier);
uv_run(&pData->loop, UV_RUN_DEFAULT);
err = uv_loop_close(&pData->loop);
while (err == UV_EBUSY) {
uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL);
uv_run(&pData->loop, UV_RUN_DEFAULT);
err = uv_loop_close(&pData->loop);
}
return;
}
int32_t dmStartUdfd(SDnode *pDnode) {
SUdfdData *pData = &pDnode->udfdData;
if (pData->startCalled) {
dInfo("dnode-mgmt start udfd already called");
return 0;
}
uv_barrier_init(&pData->barrier, 2);
pData->stopping = 0;
uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
uv_barrier_wait(&pData->barrier);
pData->startCalled = true;
pData->needCleanUp = true;
return pData->spawnErr;
}
int32_t dmStopUdfd(SDnode *pDnode) {
dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d",
pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr);
SUdfdData *pData = &pDnode->udfdData;
if (!pData->needCleanUp) {
return 0;
}
atomic_store_8(&pData->stopping, 1);
uv_barrier_destroy(&pData->barrier);
if (pData->spawnErr == 0) {
uv_process_kill(&pData->process, SIGINT);
}
uv_stop(&pData->loop);
uv_thread_join(&pData->thread);
atomic_store_8(&pData->stopping, 0);
return 0;
}
static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
dInfo("dnode-mgmt start to init");
SDnode *pDnode = pWrapper->pDnode;
......@@ -247,6 +367,10 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
}
dmReportStartup(pDnode, "dnode-transport", "initialized");
if (dmStartUdfd(pDnode) != 0) {
dError("failed to start udfd");
}
dInfo("dnode-mgmt is initialized");
return 0;
}
......@@ -254,6 +378,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
static void dmCleanupMgmt(SMgmtWrapper *pWrapper) {
dInfo("dnode-mgmt start to clean up");
SDnode *pDnode = pWrapper->pDnode;
dmStopUdfd(pDnode);
dmStopWorker(pDnode);
taosWLockLatch(&pDnode->data.latch);
......
source/dnode/mgmt/interface/inc/dmDef.h
浏览文件 @ 312b7a4f
......@@ -16,6 +16,7 @@
#ifndef _TD_DM_DEF_H_
#define _TD_DM_DEF_H_
#include "uv.h"
#include "dmLog.h"
#include "cJSON.h"
......@@ -142,6 +143,18 @@ typedef struct {
char desc[TSDB_STEP_DESC_LEN];
} SStartupInfo;
typedef struct SUdfdData {
bool startCalled;
bool needCleanUp;
uv_loop_t loop;
uv_thread_t thread;
uv_barrier_t barrier;
uv_process_t process;
int spawnErr;
int8_t stopping;
uv_pipe_t ctrlPipe;
} SUdfdData;
typedef struct SDnode {
EDndProcType ptype;
EDndNodeType ntype;
......@@ -150,6 +163,7 @@ typedef struct SDnode {
SStartupInfo startup;
SDnodeTrans trans;
SDnodeData data;
SUdfdData udfdData;
TdThreadMutex mutex;
SMgmtWrapper wrappers[NODE_END];
} SDnode;
......
source/dnode/mgmt/test/sut/src/sut.cpp
浏览文件 @ 312b7a4f
......@@ -89,7 +89,6 @@ int32_t Testbase::SendShowReq(int8_t showType, const char *tb, const char* db) {
}
SRetrieveTableReq retrieveReq = {0};
retrieveReq.type = showType;
strcpy(retrieveReq.db, db);
strcpy(retrieveReq.tb, tb);
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ 312b7a4f
......@@ -391,7 +391,6 @@ typedef struct {
int16_t numOfColumns;
int32_t rowSize;
int32_t numOfRows;
int32_t payloadLen;
void* pIter;
SMnode* pMnode;
STableMetaRsp* pMeta;
......
source/dnode/mnode/impl/src/mndDb.c
浏览文件 @ 312b7a4f
......@@ -851,6 +851,8 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) {
cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow;
cfgRsp.streamMode = pDb->cfg.streamMode;
cfgRsp.singleSTable = pDb->cfg.singleSTable;
cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions;
cfgRsp.pRetensions = pDb->cfg.pRetensions;
int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp);
void *pRsp = rpcMallocCont(contLen);
......
source/dnode/mnode/impl/src/mndFunc.c
浏览文件 @ 312b7a4f
......@@ -427,7 +427,6 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) {
SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName);
if (pFunc == NULL) {
terrno = TSDB_CODE_MND_INVALID_FUNC;
goto RETRIEVE_FUNC_OVER;
}
......@@ -439,6 +438,10 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) {
funcInfo.outputLen = pFunc->outputLen;
funcInfo.bufSize = pFunc->bufSize;
funcInfo.signature = pFunc->signature;
if (retrieveReq.ignoreCodeComment) {
funcInfo.commentSize = 0;
funcInfo.codeSize = 0;
} else {
funcInfo.commentSize = pFunc->commentSize;
funcInfo.codeSize = pFunc->codeSize;
funcInfo.pCode = taosMemoryCalloc(1, funcInfo.codeSize);
......@@ -455,6 +458,7 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) {
}
memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize);
}
}
taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo);
mndReleaseFunc(pMnode, pFunc);
}
......@@ -518,11 +522,16 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)b1, false);
if (pFunc->pComment) {
char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]);
STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)b2, false);
} else {
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, NULL, true);
}
int32_t isAgg = (pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE) ? 1 : 0;
......
source/dnode/mnode/impl/src/mndInfoSchema.c
浏览文件 @ 312b7a4f
......@@ -98,7 +98,7 @@ static const SInfosTableSchema userFuncSchema[] = {
{.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "comment", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
......
source/dnode/mnode/impl/src/mndShow.c
浏览文件 @ 312b7a4f
......@@ -18,7 +18,7 @@
#define SHOW_STEP_SIZE 100
static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq);
static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq);
static void mndFreeShowObj(SShowObj *pShow);
static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId);
static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove);
......@@ -47,18 +47,80 @@ void mndCleanupShow(SMnode *pMnode) {
}
}
static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) {
static int32_t convertToRetrieveType(char* name, int32_t len) {
int32_t type = -1;
if (strncasecmp(name, TSDB_INS_TABLE_DNODES, len) == 0) {
type = TSDB_MGMT_TABLE_DNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, len) == 0) {
type = TSDB_MGMT_TABLE_MNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, len) == 0) {
type = TSDB_MGMT_TABLE_MODULE;
} else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, len) == 0) {
type = TSDB_MGMT_TABLE_QNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, len) == 0) {
type = TSDB_MGMT_TABLE_BNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) {
type = TSDB_MGMT_TABLE_SNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) {
type = TSDB_MGMT_TABLE_CLUSTER;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, len) == 0) {
type = TSDB_MGMT_TABLE_DB;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) {
type = TSDB_MGMT_TABLE_FUNC;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) {
// type = TSDB_MGMT_TABLE_INDEX;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) {
type = TSDB_MGMT_TABLE_STB;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, len) == 0) {
type = TSDB_MGMT_TABLE_STREAMS;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) {
type = TSDB_MGMT_TABLE_TABLE;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) {
// type = TSDB_MGMT_TABLE_DIST;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, len) == 0) {
type = TSDB_MGMT_TABLE_USER;
} else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, len) == 0) {
type = TSDB_MGMT_TABLE_GRANTS;
} else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) {
type = TSDB_MGMT_TABLE_VGROUP;
} else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, len) == 0) {
type = TSDB_MGMT_TABLE_TOPICS;
} else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) {
type = TSDB_MGMT_TABLE_CONSUMERS;
} else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) {
type = TSDB_MGMT_TABLE_SUBSCRIBES;
} else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, len) == 0) {
type = TSDB_MGMT_TABLE_TRANS;
} else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, len) == 0) {
type = TSDB_MGMT_TABLE_SMAS;
} else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, len) == 0) {
type = TSDB_MGMT_TABLE_CONFIGS;
} else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, len) == 0) {
type = TSDB_MGMT_TABLE_CONNS;
} else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, len) == 0) {
type = TSDB_MGMT_TABLE_QUERIES;
} else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) {
type = TSDB_MGMT_TABLE_VNODES;
} else {
// ASSERT(0);
}
return type;
}
static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) {
SShowMgmt *pMgmt = &pMnode->showMgmt;
int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1);
if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1);
int32_t size = sizeof(SShowObj) + pReq->payloadLen;
int32_t size = sizeof(SShowObj);
SShowObj showObj = {0};
showObj.id = showId;
showObj.pMnode = pMnode;
showObj.type = pReq->type;
showObj.payloadLen = pReq->payloadLen;
showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb));
memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN);
int32_t keepTime = tsShellActivityTimer * 6 * 1000;
......@@ -127,10 +189,6 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
}
if (retrieveReq.showId == 0) {
SShowReq req = {0};
req.type = retrieveReq.type;
strncpy(req.db, retrieveReq.db, tListLen(req.db));
STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1);
if (pMeta == NULL) {
terrno = TSDB_CODE_MND_INVALID_INFOS_TBL;
......@@ -138,7 +196,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
return -1;
}
pShow = mndCreateShowObj(pMnode, &req);
pShow = mndCreateShowObj(pMnode, &retrieveReq);
if (pShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta req since %s", terrstr());
......
source/dnode/mnode/impl/src/mndStb.c
浏览文件 @ 312b7a4f
......@@ -20,6 +20,7 @@
#include "mndInfoSchema.h"
#include "mndMnode.h"
#include "mndPerfSchema.h"
#include "mndScheduler.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
......@@ -443,6 +444,25 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) {
*(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod;
}
if (pStb->ast1Len > 0) {
if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pRSmaParam->pFuncIds);
taosMemoryFreeClear(req.stbCfg.pRSmaParam);
taosMemoryFreeClear(req.stbCfg.pSchema);
return NULL;
}
}
if (pStb->ast2Len > 0) {
int32_t qmsgLen2 = 0;
if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pRSmaParam->pFuncIds);
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(req.stbCfg.pRSmaParam);
taosMemoryFreeClear(req.stbCfg.pSchema);
return NULL;
}
}
req.stbCfg.pRSmaParam = pRSmaParam;
}
......@@ -451,6 +471,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
if (pHead == NULL) {
if (pRSmaParam) {
taosMemoryFreeClear(pRSmaParam->pFuncIds);
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(pRSmaParam->qmsg2);
taosMemoryFreeClear(pRSmaParam);
}
taosMemoryFreeClear(req.stbCfg.pSchema);
......@@ -467,6 +489,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
*pContLen = contLen;
if (pRSmaParam) {
taosMemoryFreeClear(pRSmaParam->pFuncIds);
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(pRSmaParam->qmsg2);
taosMemoryFreeClear(pRSmaParam);
}
taosMemoryFreeClear(req.stbCfg.pSchema);
......
source/dnode/mnode/impl/test/func/func.cpp
浏览文件 @ 312b7a4f
......@@ -240,7 +240,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST);
}
{
......@@ -371,7 +371,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST);
}
}
......
source/dnode/mnode/impl/test/show/show.cpp
浏览文件 @ 312b7a4f
......@@ -36,7 +36,7 @@ TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG);
ASSERT_NE(pRsp->code, 0);
}
TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
......@@ -50,7 +50,7 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG);
ASSERT_NE(pRsp->code, 0);
}
TEST_F(MndTestShow, 03_ShowMsg_Conn) {
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ 312b7a4f
......@@ -391,6 +391,7 @@ struct SReadH {
#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
#define TSDB_READ_FSET(rh) (&((rh)->rSet))
#define TSDB_READ_TABLE(rh) ((rh)->pTable)
#define TSDB_READ_TABLE_UID(rh) ((rh)->pTable->uid)
#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
......
source/dnode/vnode/src/meta/metaTDBImpl.c
浏览文件 @ 312b7a4f
......@@ -626,14 +626,8 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
#ifdef META_TDB_SMA_TEST
STSmaWrapper *pSW = NULL;
pSW = taosMemoryCalloc(1, sizeof(*pSW));
if (pSW == NULL) {
return NULL;
}
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
if (pCur == NULL) {
taosMemoryFree(pSW);
return NULL;
}
......@@ -653,6 +647,12 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
continue;
}
if ((pSW == NULL) && ((pSW = taosMemoryCalloc(1, sizeof(*pSW))) == NULL)) {
TDB_FREE(pSmaVal);
metaCloseSmaCursor(pCur);
return NULL;
}
++pSW->number;
STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
if (tptr == NULL) {
......
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ 312b7a4f
......@@ -3260,6 +3260,9 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat
return TSDB_CODE_SUCCESS;
}
tsdbDebug("vgId:%d succeed to load block statis part for uid %" PRIu64, REPO_ID(pHandle->pTsdb),
TSDB_READ_TABLE_UID(&pHandle->rhelper));
int16_t* colIds = pHandle->defaultLoadColumn->pData;
size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle);
......
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
浏览文件 @ 312b7a4f
......@@ -322,15 +322,18 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
ASSERT(pBlock->numOfSubBlocks <= 1);
if (!pBlock->aggrStat) {
tsdbDebug("vgId:%d no need to load block statis part for uid %" PRIu64 " since not exist", REPO_ID(pReadh->pRepo),
TSDB_READ_TABLE_UID(pReadh));
return TSDB_STATIS_NONE;
}
SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh);
if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) {
tsdbError("vgId:%d failed to load block aggr part while seek file %s to offset %" PRIu64 " since %s",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset,
tstrerror(terrno));
tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64 " while seek file %s to offset %" PRIu64
" since %s",
TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
(uint64_t)pBlock->aggrOffset, tstrerror(terrno));
return -1;
}
......@@ -339,25 +342,28 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr);
if (nreadAggr < 0) {
tsdbError("vgId:%d failed to load block aggr part while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), tstrerror(terrno),
(uint64_t)pBlock->aggrOffset, sizeAggr);
tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64
" while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
tstrerror(terrno), (uint64_t)pBlock->aggrOffset, sizeAggr);
return -1;
}
if (nreadAggr < sizeAggr) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d block aggr part in file %s is corrupted, offset:%" PRIu64 " expected bytes:%" PRIzu
" read bytes: %" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr,
nreadAggr);
tsdbError("vgId:%d block statis part for uid %" PRIu64 " in file %s is corrupted, offset:%" PRIu64
" expected bytes:%" PRIzu " read bytes: %" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
(uint64_t)pBlock->aggrOffset, sizeAggr, nreadAggr);
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d block aggr part in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr);
tsdbError("vgId:%d block statis part for uid %" PRIu64
"in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
(uint64_t)pBlock->aggrOffset, sizeAggr);
return -1;
}
return 0;
......@@ -367,7 +373,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
ASSERT(pBlock->numOfSubBlocks <= 1);
SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s",
tsdbError("vgId:%d failed to load block head part while seek file %s to offset %" PRId64 " since %s",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
return -1;
}
......@@ -377,14 +383,14 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size);
if (nread < 0) {
tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu,
tsdbError("vgId:%d failed to load block head part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size);
return -1;
}
if (nread < size) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu
tsdbError("vgId:%d block head part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu
" read bytes: %" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread);
return -1;
......@@ -392,7 +398,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu,
tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size);
return -1;
}
......@@ -546,7 +552,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize);
return -1;
}
......
source/dnode/vnode/src/tsdb/tsdbSma.c
浏览文件 @ 312b7a4f
......@@ -643,6 +643,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers
SSubmitMsgIter msgIter = {0};
SSubmitBlk *pBlock = NULL;
SInterval interval = {0};
TSKEY lastWinSKey = INT64_MIN;
if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
......@@ -657,7 +658,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers
SSubmitBlkIter blkIter = {0};
if (tInitSubmitBlkIter(pBlock, &blkIter) != TSDB_CODE_SUCCESS) {
tdFreeTSmaWrapper(pSW);
pSW = tdFreeTSmaWrapper(pSW);
break;
}
......@@ -667,16 +668,19 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers
tdFreeTSmaWrapper(pSW);
break;
}
if (pSW == NULL) {
if (!pSW || (pTSma->tableUid != pBlock->suid)) {
if (pSW) {
pSW = tdFreeTSmaWrapper(pSW);
}
if ((pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) {
break;
}
if ((pSW->number) <= 0 || (pSW->tSma == NULL)) {
tdFreeTSmaWrapper(pSW);
pSW = tdFreeTSmaWrapper(pSW);
break;
}
pTSma = pSW->tSma;
}
interval.interval = pTSma->interval;
interval.intervalUnit = pTSma->intervalUnit;
......@@ -684,14 +688,17 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers
interval.precision = REPO_CFG(pTsdb)->precision;
interval.sliding = pTSma->sliding;
interval.slidingUnit = pTSma->slidingUnit;
}
TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision);
if (lastWinSKey != winSKey) {
lastWinSKey = winSKey;
tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version);
// TODO: release only when suid changes.
tdDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW);
} else {
tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated",
REPO_ID(pTsdb), pTSma->indexUid, winSKey);
}
}
}
......
source/dnode/vnode/src/vnd/vnodeSvr.c
浏览文件 @ 312b7a4f
......@@ -215,10 +215,21 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) {
return -1;
}
// TODO: remove the debug log
SRSmaParam *param = vCreateTbReq.stbCfg.pRSmaParam;
if (param) {
printf("qmsg1 len = %d, body = %s\n", param->qmsg1 ? (int32_t)strlen(param->qmsg1) : 0,
param->qmsg1 ? param->qmsg1 : "");
printf("qmsg1 len = %d, body = %s\n", param->qmsg2 ? (int32_t)strlen(param->qmsg2) : 0,
param->qmsg2 ? param->qmsg2 : "");
}
taosMemoryFree(vCreateTbReq.stbCfg.pSchema);
taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema);
if (vCreateTbReq.stbCfg.pRSmaParam) {
taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds);
taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1);
taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2);
taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam);
}
taosMemoryFree(vCreateTbReq.name);
......
source/libs/catalog/src/catalog.c
浏览文件 @ 312b7a4f
......@@ -643,6 +643,49 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *funcName, SFuncInfo **out) {
char *msg = NULL;
int32_t msgLen = 0;
ctgDebug("try to get udf info from mnode, funcName:%s", funcName);
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]((void *)funcName, &msg, 0, &msgLen);
if (code) {
ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName);
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_RETRIEVE_FUNC,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) {
ctgDebug("funcName %s not exist in mnode", funcName);
taosMemoryFreeClear(*out);
CTG_RET(TSDB_CODE_SUCCESS);
}
ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName);
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)](*out, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process get udf rsp failed, code:%x, funcName:%s", code, funcName);
CTG_ERR_RET(code);
}
ctgDebug("Got udf from mnode, funcName:%s", funcName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) {
if (NULL == pCtg->dbCache) {
......@@ -2811,6 +2854,30 @@ int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps,
CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo));
}
int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
*pInfo = taosMemoryMalloc(sizeof(SFuncInfo));
if (NULL == *pInfo) {
CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo));
_return:
if (code) {
taosMemoryFreeClear(*pInfo);
}
CTG_API_LEAVE(code);
}
void catalogDestroy(void) {
qInfo("start to destroy catalog");
......
source/libs/executor/inc/executorimpl.h
浏览文件 @ 312b7a4f
......@@ -382,7 +382,7 @@ typedef struct SSysTableScanInfo {
void* pCur; // cursor for iterate the local table meta store.
SArray* scanCols; // SArray<int16_t> scan column id list
int32_t type; // show type, TODO remove it
// int32_t type; // show type, TODO remove it
SName name;
SSDataBlock* pRes;
int32_t capacity;
......@@ -479,7 +479,7 @@ typedef struct {
typedef struct SGroupbyOperatorInfo {
SOptrBasicInfo binfo;
SArray* pGroupCols;
SArray* pGroupCols; // group by columns, SArray<SColumn>
SArray* pGroupColVals; // current group column values, SArray<SGroupKeys>
SNode* pCondition;
bool isInit; // denote if current val is initialized or not
......@@ -600,7 +600,6 @@ typedef struct SJoinOperatorInfo {
int32_t rightPos;
SColumnInfo rightCol;
SNode *pOnCondition;
// SRspResultInfo resultInfo;
} SJoinOperatorInfo;
int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
......
source/libs/executor/src/executorimpl.c
浏览文件 @ 312b7a4f
......@@ -1087,6 +1087,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
pCtx[i].currentStage = MAIN_SCAN;
SInputColumnInfoData* pInput = &pCtx[i].input;
pInput->uid = pBlock->info.uid;
SExprInfo* pOneExpr = &pOperator->pExpr[i];
for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
......@@ -1101,7 +1102,9 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column
ASSERT(pInput->pData[j] != NULL);
} else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
if (createDummyCol) {
// todo avoid case: top(k, 12), 12 is the value parameter.
// sum(11), 11 is also the value parameter.
if (createDummyCol && pOneExpr->base.numOfParams == 1) {
code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows);
if (code != TSDB_CODE_SUCCESS) {
return code;
......@@ -1876,8 +1879,10 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
}
}
pCtx->resDataInfo.interBufSize = env.calcMemSize;
} else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR || pExpr->pExpr->nodeType == QUERY_NODE_VALUE) {
pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; // for simple column, the intermediate buffer needs to hold one element.
} else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR ||
pExpr->pExpr->nodeType == QUERY_NODE_VALUE) {
// for simple column, the intermediate buffer needs to hold one element.
pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes;
}
pCtx->input.numOfInputCols = pFunct->numOfParams;
......@@ -1890,53 +1895,42 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
pCtx->order = TSDB_ORDER_ASC;
pCtx->start.key = INT64_MIN;
pCtx->end.key = INT64_MIN;
#if 0
for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
// int16_t type = pFunct->param[j].nType;
// int16_t bytes = pFunct->param[j].nLen;
// if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
// taosVariantCreateFromBinary(&pCtx->param[j], pFunct->param[j].pz, bytes, type);
// } else {
// taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pFunct->param[j].i, bytes, type);
// }
}
// set the order information for top/bottom query
int32_t functionId = pCtx->functionId;
if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
int32_t f = getExprFunctionId(&pExpr[0]);
assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY);
// pCtx->param[2].i = pQueryAttr->order.order;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[3].i = functionId;
pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT;
// pCtx->param[1].i = pQueryAttr->order.col.info.colId;
} else if (functionId == FUNCTION_INTERP) {
// pCtx->param[2].i = (int8_t)pQueryAttr->fillType;
// if (pQueryAttr->fillVal != NULL) {
// if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) {
// pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
// } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value
// if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
// taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType);
// }
pCtx->numOfParams = pExpr->base.numOfParams;
pCtx->param = pFunct->pParam;
// for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
// // set the order information for top/bottom query
// int32_t functionId = pCtx->functionId;
// if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
// int32_t f = getExprFunctionId(&pExpr[0]);
// assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY);
//
// // pCtx->param[2].i = pQueryAttr->order.order;
// // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
// // pCtx->param[3].i = functionId;
// // pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT;
//
// // pCtx->param[1].i = pQueryAttr->order.col.info.colId;
// } else if (functionId == FUNCTION_INTERP) {
// // pCtx->param[2].i = (int8_t)pQueryAttr->fillType;
// // if (pQueryAttr->fillVal != NULL) {
// // if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) {
// // pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
// // } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value
// // if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
// // taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType);
// // }
// // }
// // }
// } else if (functionId == FUNCTION_TWA) {
// // pCtx->param[1].i = pQueryAttr->window.skey;
// // pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT;
// // pCtx->param[2].i = pQueryAttr->window.ekey;
// // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
// } else if (functionId == FUNCTION_ARITHM) {
// // pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
// }
// }
} else if (functionId == FUNCTION_TS_COMP) {
// pCtx->param[0].i = pQueryAttr->vgId; //TODO this should be the parameter from client
pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT;
} else if (functionId == FUNCTION_TWA) {
// pCtx->param[1].i = pQueryAttr->window.skey;
pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT;
// pCtx->param[2].i = pQueryAttr->window.ekey;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
} else if (functionId == FUNCTION_ARITHM) {
// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
}
#endif
}
for (int32_t i = 1; i < numOfOutput; ++i) {
......@@ -1955,7 +1949,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
for (int32_t i = 0; i < numOfOutput; ++i) {
for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) {
taosVariantDestroy(&pCtx[i].param[j]);
taosVariantDestroy(&pCtx[i].param[j].param);
}
taosVariantDestroy(&pCtx[i].tag);
......@@ -6487,9 +6481,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t numOfCols = 0;
tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId);
if (pDataReader == NULL) {
return NULL;
}
SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);
......
source/libs/executor/src/scanoperator.c
浏览文件 @ 312b7a4f
......@@ -319,7 +319,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// The read handle is not initialized yet, since no qualified tables exists
if (pTableScanInfo->dataReader == NULL) {
if (pTableScanInfo->dataReader == NULL || pOperator->status == OP_EXEC_DONE) {
return NULL;
}
......@@ -375,10 +375,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
return p;
}
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput,
int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime,
SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition,
SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag,
int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock,
SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
assert(repeatTime > 0);
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
......@@ -393,10 +392,10 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
pInfo->interval = *pInterval;
pInfo->sampleRatio = sampleRatio;
pInfo->dataBlockLoadFlag = dataLoadFlag;
pInfo->dataBlockLoadFlag= dataLoadFlag;
pInfo->pResBlock = pResBlock;
pInfo->pFilterNode = pCondition;
pInfo->dataReader = pTsdbReadHandle;
pInfo->dataReader = pDataReader;
pInfo->times = repeatTime;
pInfo->reverseTimes = reverseTime;
pInfo->order = order;
......@@ -677,7 +676,8 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) {
tsem_destroy(&pInfo->ready);
blockDataDestroy(pInfo->pRes);
if (pInfo->type == TSDB_MGMT_TABLE_TABLE) {
const char* name = tNameGetTableName(&pInfo->name);
if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
metaCloseTbCursor(pInfo->pCur);
}
}
......@@ -812,7 +812,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
SSysTableScanInfo* pInfo = pOperator->info;
// retrieve local table list info from vnode
if (pInfo->type == TSDB_MGMT_TABLE_TABLE) {
const char* name = tNameGetTableName(&pInfo->name);
if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
if (pInfo->pCur == NULL) {
pInfo->pCur = metaOpenTbCursor(pInfo->readHandle);
}
......@@ -864,8 +865,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
while (1) {
int64_t startTs = taosGetTimestampUs();
pInfo->req.type = pInfo->type;
strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
if (pInfo->showRewrite) {
......@@ -947,68 +946,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
pInfo->pCondition = pCondition;
pInfo->scanCols = colList;
// TODO remove it
int32_t tableType = 0;
const char* name = tNameGetTableName(pName);
if (strncasecmp(name, TSDB_INS_TABLE_DNODES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_DNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_MNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_MODULE;
} else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_QNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_BNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_SNODE;
} else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_CLUSTER;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_DB;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_FUNC;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, tListLen(pName->tname)) == 0) {
// tableType = TSDB_MGMT_TABLE_INDEX;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_STB;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_STREAMS;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_TABLE;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, tListLen(pName->tname)) == 0) {
// tableType = TSDB_MGMT_TABLE_DIST;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_USER;
} else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_GRANTS;
} else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_VGROUP;
} else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_TOPICS;
} else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_CONSUMERS;
} else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_SUBSCRIBES;
} else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_TRANS;
} else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_SMAS;
} else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_CONFIGS;
} else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_CONNS;
} else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_QUERIES;
} else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, tListLen(pName->tname)) == 0) {
tableType = TSDB_MGMT_TABLE_VNODES;
} else {
ASSERT(0);
}
tNameAssign(&pInfo->name, pName);
pInfo->type = tableType;
if (pInfo->type == TSDB_MGMT_TABLE_TABLE) {
const char* name = tNameGetTableName(&pInfo->name);
if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
pInfo->readHandle = pSysTableReadHandle;
blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity);
} else {
......
source/libs/function/inc/builtinsimpl.h
浏览文件 @ 312b7a4f
......@@ -58,6 +58,9 @@ bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t firstFunction(SqlFunctionCtx *pCtx);
int32_t lastFunction(SqlFunctionCtx *pCtx);
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
int32_t topFunction(SqlFunctionCtx *pCtx);
#ifdef __cplusplus
}
#endif
......
source/libs/function/inc/tudf.h
浏览文件 @ 312b7a4f
......@@ -27,41 +27,31 @@ extern "C" {
#endif
#define UDF_LISTEN_PIPE_NAME_LEN 32
#define UDF_LISTEN_PIPE_NAME_PREFIX "udf.sock."
#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock."
//======================================================================================
//begin API to taosd and qworker
enum {
UDFC_CODE_STOPPING = -1,
UDFC_CODE_RESTARTING = -2,
UDFC_CODE_PIPE_READ_ERR = -3,
};
/*TODO: no api for dnode startudfd/stopudfd*/
/**
* start udfd dameon service
*/
int32_t startUdfd(int32_t dnodeId);
/**
* stop udfd dameon service
*/
int32_t stopUdfd(int32_t dnodeId);
typedef void *UdfcHandle;
typedef void *UdfcFuncHandle;
/**
* create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
* @return error code
*/
int32_t createUdfdProxy(int32_t dnodeId);
int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle);
/**
* destroy udfd proxy
* @return error code
*/
int32_t destroyUdfdProxy(int32_t dnodeId);
int32_t udfcClose(UdfcHandle proxyhandle);
typedef void *UdfHandle;
/**
* setup udf
......@@ -69,7 +59,7 @@ typedef void *UdfHandle;
* @param handle, out
* @return error code
*/
int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle);
int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle);
typedef struct SUdfColumnMeta {
int16_t type;
......@@ -116,26 +106,26 @@ typedef struct SUdfInterBuf {
} SUdfInterBuf;
// output: interBuf
int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf);
int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf);
// input: block, state
// output: newState
int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
// input: interBuf
// output: resultData
int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
// input: interbuf1, interbuf2
// output: resultBuf
int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
// input: block
// output: resultData
int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData);
int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData);
/**
* tearn down udf
* @param handle
* @return
*/
int32_t teardownUdf(UdfHandle handle);
int32_t teardownUdf(UdfcFuncHandle handle);
// end API to taosd and qworker
//=============================================================================================================================
......
source/libs/function/inc/udfc.h
浏览文件 @ 312b7a4f
......@@ -30,20 +30,20 @@ typedef struct SUdfInfo {
char *path;
} SUdfInfo;
typedef void *UdfHandle;
typedef void *UdfcFuncHandle;
int32_t createUdfdProxy();
int32_t destroyUdfdProxy();
//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles);
//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfcFuncHandle *handles);
int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle);
int32_t setupUdf(SUdfInfo* udf, UdfcFuncHandle* handle);
int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate,
int32_t callUdf(UdfcFuncHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate,
int32_t *newStateSize, SSDataBlock *output);
int32_t teardownUdf(UdfHandle handle);
int32_t teardownUdf(UdfcFuncHandle handle);
typedef struct SUdfSetupRequest {
char udfName[16]; //
......
source/libs/function/src/builtins.c
浏览文件 @ 312b7a4f
......@@ -193,8 +193,15 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
return TSDB_CODE_SUCCESS;
}
static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType = (SDataType){.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}
static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// todo
SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type};
return TSDB_CODE_SUCCESS;
}
......@@ -497,9 +504,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.type = FUNCTION_TYPE_TOP,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateTop,
.getEnvFunc = getMinmaxFuncEnv,
.initFunc = maxFunctionSetup,
.processFunc = maxFunction,
.getEnvFunc = getTopBotFuncEnv,
.initFunc = functionSetup,
.processFunc = topFunction,
.finalizeFunc = functionFinalize
},
{
......@@ -876,7 +883,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "tbname",
.type = FUNCTION_TYPE_TBNAME,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
.translateFunc = NULL,
.translateFunc = translateTbnameColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
......@@ -914,7 +921,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
},
{
.name = "_wendts",
.type = FUNCTION_TYPE_QENDTS,
.type = FUNCTION_TYPE_WENDTS,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
.translateFunc = translateTimePseudoColumn,
.getEnvFunc = getTimePseudoFuncEnv,
......
source/libs/function/src/builtinsimpl.c
浏览文件 @ 312b7a4f
......@@ -14,10 +14,11 @@
*/
#include "builtinsimpl.h"
#include "tpercentile.h"
#include <libs/nodes/querynodes.h>
#include "querynodes.h"
#include "taggfunction.h"
#include "tdatablock.h"
#include "tpercentile.h"
#define SET_VAL(_info, numOfElem, res) \
do { \
......@@ -472,17 +473,6 @@ int32_t maxFunction(SqlFunctionCtx *pCtx) {
return TSDB_CODE_SUCCESS;
}
typedef struct STopBotRes {
int32_t num;
} STopBotRes;
bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0);
int32_t bytes = pColNode->node.resType.bytes;
SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1);
return true;
}
typedef struct SStddevRes {
double result;
int64_t count;
......@@ -523,7 +513,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
switch (type) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t* plist = (int8_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
for (int32_t i = start; i < numOfRows + start; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
......@@ -749,9 +739,9 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) {
return TSDB_CODE_SUCCESS;
}
// TODO set the correct parameter.
void percentileFinalize(SqlFunctionCtx* pCtx) {
double v = 50;//pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey;
SVariant* pVal = &pCtx->param[1].param;
double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d;
SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo);
......@@ -1173,3 +1163,130 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) {
}
}
typedef struct STopBotResItem {
SVariant v;
uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data
struct {
int32_t pageId;
int32_t offset;
} tuplePos; // tuple data of this chosen row
} STopBotResItem;
typedef struct STopBotRes {
int32_t num;
STopBotResItem *pItems;
} STopBotRes;
bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0);
int32_t bytes = pColNode->node.resType.bytes;
SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1);
pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * bytes;
return true;
}
static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) {
SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
STopBotRes* pRes = GET_ROWCELL_INTERBUF(pResInfo);
pRes->pItems = (STopBotResItem*)((char*) pRes + sizeof(STopBotRes));
return pRes;
}
static void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid);
int32_t topFunction(SqlFunctionCtx *pCtx) {
int32_t numOfElems = 0;
STopBotRes *pRes = getTopBotOutputInfo(pCtx);
assert(pRes->num >= 0);
// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) {
// buildTopBotStruct(pRes, pCtx);
// }
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
int32_t type = pInput->pData[0]->info.type;
int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows;
for (int32_t i = start; i < numOfRows + start; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElems++;
char* data = colDataGetData(pCol, i);
doAddIntoResult(pRes, pCtx->param[1].param.i, data, type, pInput->uid);
}
// treat the result as only one result
SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1);
return TSDB_CODE_SUCCESS;
}
static int32_t topBotResComparFn(const void *p1, const void *p2, const void *param) {
uint16_t type = *(uint16_t *) param;
STopBotResItem *val1 = (STopBotResItem *) p1;
STopBotResItem *val2 = (STopBotResItem *) p2;
if (IS_SIGNED_NUMERIC_TYPE(type)) {
if (val1->v.i == val2->v.i) {
return 0;
}
return (val1->v.i > val2->v.i) ? 1 : -1;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
if (val1->v.u == val2->v.u) {
return 0;
}
return (val1->v.u > val2->v.u) ? 1 : -1;
}
if (val1->v.d == val2->v.d) {
return 0;
}
return (val1->v.d > val2->v.d) ? 1 : -1;
}
void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid) {
SVariant val = {0};
taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type);
STopBotResItem *pItems = pRes->pItems;
assert(pItems != NULL);
// not full yet
if (pRes->num < maxSize) {
STopBotResItem* pItem = &pItems[pRes->num];
pItem->v = val;
pItem->uid = uid;
pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer
pRes->num++;
taosheapsort((void *) pItem, sizeof(STopBotResItem), pRes->num, (const void *) &type, topBotResComparFn, false);
} else { // replace the minimum value in the result
if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) ||
(IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) ||
(IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) {
STopBotResItem* pItem = &pItems[pRes->num];
pItem->v = val;
pItem->uid = uid;
pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer
taosheapadjust((void *) pItem, sizeof(STopBotResItem), 0, pRes->num - 1, (const void *) &type, topBotResComparFn, NULL, false);
}
}
}
void topBotFinalize(SqlFunctionCtx* pCtx) {
functionFinalize(pCtx);
}
\ No newline at end of file
source/libs/function/src/functionMgt.c
浏览文件 @ 312b7a4f
......@@ -29,6 +29,7 @@ typedef struct SFuncMgtService {
typedef struct SUdfInfo {
SDataType outputDt;
int8_t funcType;
} SUdfInfo;
static SFuncMgtService gFunMgtService;
......@@ -52,30 +53,41 @@ static void doInitFunctionTable() {
gFunMgtService.pUdfTable = NULL;
}
static int8_t getUdfType(int32_t funcId) {
SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1);
return pUdf->funcType;
}
static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
if (fmIsUserDefinedFunc(funcId)) {
return getUdfType(funcId);
}
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification);
}
static int32_t getUdfId(const char* pFuncName) {
// todo: udf by call catalog
if (1) {
static int32_t getUdfId(SFmGetFuncInfoParam* pParam, const char* pFuncName) {
SFuncInfo* pInfo = NULL;
int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFuncName, &pInfo);
if (TSDB_CODE_SUCCESS != code || NULL == pInfo) {
return -1;
}
if (NULL == gFunMgtService.pUdfTable) {
gFunMgtService.pUdfTable = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SUdfInfo));
}
SUdfInfo info = {0}; //todo
SUdfInfo info = { .outputDt.type = pInfo->outputType, .outputDt.bytes = pInfo->outputLen, .funcType = pInfo->funcType };
taosArrayPush(gFunMgtService.pUdfTable, &info);
tFreeSFuncInfo(pInfo);
taosMemoryFree(pInfo);
return taosArrayGetSize(gFunMgtService.pUdfTable) + FUNC_UDF_ID_START_OFFSET_VAL;
}
static int32_t getFuncId(const char* pFuncName) {
static int32_t getFuncId(SFmGetFuncInfoParam* pParam, const char* pFuncName) {
void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName));
if (NULL == pVal) {
return getUdfId(pFuncName);
return getUdfId(pParam, pFuncName);
}
return *(int32_t*)pVal;
}
......@@ -91,8 +103,8 @@ int32_t fmFuncMgtInit() {
return initFunctionCode;
}
int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) {
*pFuncId = getFuncId(pFuncName);
int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) {
*pFuncId = getFuncId(pParam, pFuncName);
if (*pFuncId < 0) {
return TSDB_CODE_FAILED;
}
......
source/libs/function/src/tudf.c
浏览文件 @ 312b7a4f
......@@ -14,19 +14,15 @@
*/
#include "uv.h"
#include "os.h"
#include "tlog.h"
#include "tudf.h"
#include "tudfInt.h"
#include "tarray.h"
#include "tdatablock.h"
//TODO: when startup, set thread poll size. add it to cfg
//TODO: test for udfd restart
//TODO: udfd restart when exist or aborts
//TODO: deal with uv task that has been started and then udfd core dumped
//TODO: network error processing.
//TODO: add unit test
//TODO: include all global variable under context struct
/* Copyright (c) 2013, Ben Noordhuis <info@bnoordhuis.nl>
* The QUEUE is copied from queue.h under libuv
* */
......@@ -125,12 +121,35 @@ enum {
UV_TASK_DISCONNECT = 2
};
int64_t gUdfTaskSeqNum = 0;
typedef struct SUdfdProxy {
int32_t dnodeId;
uv_barrier_t gUdfInitBarrier;
uv_loop_t gUdfdLoop;
uv_thread_t gUdfLoopThread;
uv_async_t gUdfLoopTaskAync;
uv_async_t gUdfLoopStopAsync;
uv_mutex_t gUdfTaskQueueMutex;
int8_t gUdfcState;
QUEUE gUdfTaskQueue;
QUEUE gUvProcTaskQueue;
// int8_t gUdfcState = UDFC_STATE_INITAL;
// QUEUE gUdfTaskQueue = {0};
// QUEUE gUvProcTaskQueue = {0};
} SUdfdProxy;
typedef struct SUdfUvSession {
SUdfdProxy *udfc;
int64_t severHandle;
uv_pipe_t *udfSvcPipe;
} SUdfUvSession;
typedef struct SClientUvTaskNode {
SUdfdProxy *udfc;
int8_t type;
int errCode;
......@@ -169,7 +188,6 @@ typedef struct SClientUdfTask {
} _teardown;
};
} SClientUdfTask;
typedef struct SClientConnBuf {
......@@ -185,34 +203,13 @@ typedef struct SClientUvConn {
SClientConnBuf readBuf;
} SClientUvConn;
uv_process_t gUdfdProcess;
uv_barrier_t gUdfInitBarrier;
uv_loop_t gUdfdLoop;
uv_thread_t gUdfLoopThread;
uv_async_t gUdfLoopTaskAync;
uv_async_t gUdfLoopStopAsync;
uv_mutex_t gUdfTaskQueueMutex;
int64_t gUdfTaskSeqNum = 0;
enum {
UDFC_STATE_INITAL = 0, // initial state
UDFC_STATE_STARTNG, // starting after createUdfdProxy
UDFC_STATE_STARTNG, // starting after udfcOpen
UDFC_STATE_READY, // started and begin to receive quests
UDFC_STATE_RESTARTING, // udfd abnormal exit. cleaning up and restart.
UDFC_STATE_STOPPING, // stopping after destroyUdfdProxy
UDFC_STATE_STOPPING, // stopping after udfcClose
UDFC_STATUS_FINAL, // stopped
};
int8_t gUdfcState = UDFC_STATE_INITAL;
//double circular linked list
QUEUE gUdfTaskQueue = {0};
QUEUE gUvProcTaskQueue = {0};
int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) {
int32_t len = 0;
......@@ -777,13 +774,14 @@ void onUdfClientConnect(uv_connect_t *connect, int status) {
int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) {
SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode));
uvTask->type = uvTaskType;
uvTask->udfc = task->session->udfc;
if (uvTaskType == UV_TASK_CONNECT) {
} else if (uvTaskType == UV_TASK_REQ_RSP) {
uvTask->pipe = task->session->udfSvcPipe;
SUdfRequest request;
request.type = task->type;
request.seqNum = gUdfTaskSeqNum++;
request.seqNum = atomic_fetch_add_64(&gUdfTaskSeqNum, 1);
if (task->type == UDF_TASK_SETUP) {
request.setup = task->_setup.req;
......@@ -815,11 +813,11 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN
int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) {
debugPrint("%s, %d", "queue uv task", uvTask->type);
uv_mutex_lock(&gUdfTaskQueueMutex);
QUEUE_INSERT_TAIL(&gUdfTaskQueue, &uvTask->recvTaskQueue);
uv_mutex_unlock(&gUdfTaskQueueMutex);
uv_async_send(&gUdfLoopTaskAync);
SUdfdProxy *udfc = uvTask->udfc;
uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue);
uv_mutex_unlock(&udfc->gUdfTaskQueueMutex);
uv_async_send(&udfc->gUdfLoopTaskAync);
uv_sem_wait(&uvTask->taskSem);
uv_sem_destroy(&uvTask->taskSem);
......@@ -832,7 +830,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
switch (uvTask->type) {
case UV_TASK_CONNECT: {
uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t));
uv_pipe_init(&gUdfdLoop, pipe, 0);
uv_pipe_init(&uvTask->udfc->gUdfdLoop, pipe, 0);
uvTask->pipe = pipe;
SClientUvConn *conn = taosMemoryMalloc(sizeof(SClientUvConn));
......@@ -873,142 +871,98 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
}
void udfClientAsyncCb(uv_async_t *async) {
SUdfdProxy *udfc = async->data;
QUEUE wq;
uv_mutex_lock(&gUdfTaskQueueMutex);
QUEUE_MOVE(&gUdfTaskQueue, &wq);
uv_mutex_unlock(&gUdfTaskQueueMutex);
uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq);
uv_mutex_unlock(&udfc->gUdfTaskQueueMutex);
while (!QUEUE_EMPTY(&wq)) {
QUEUE* h = QUEUE_HEAD(&wq);
QUEUE_REMOVE(h);
SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue);
startUvUdfTask(task);
QUEUE_INSERT_TAIL(&gUvProcTaskQueue, &task->procTaskQueue);
QUEUE_INSERT_TAIL(&udfc->gUvProcTaskQueue, &task->procTaskQueue);
}
}
void cleanUpUvTasks() {
void cleanUpUvTasks(SUdfdProxy *udfc) {
QUEUE wq;
uv_mutex_lock(&gUdfTaskQueueMutex);
QUEUE_MOVE(&gUdfTaskQueue, &wq);
uv_mutex_unlock(&gUdfTaskQueueMutex);
uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq);
uv_mutex_unlock(&udfc->gUdfTaskQueueMutex);
while (!QUEUE_EMPTY(&wq)) {
QUEUE* h = QUEUE_HEAD(&wq);
QUEUE_REMOVE(h);
SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue);
if (gUdfcState == UDFC_STATE_STOPPING) {
if (udfc->gUdfcState == UDFC_STATE_STOPPING) {
task->errCode = UDFC_CODE_STOPPING;
} else if (gUdfcState == UDFC_STATE_RESTARTING) {
task->errCode = UDFC_CODE_RESTARTING;
}
uv_sem_post(&task->taskSem);
}
// TODO: deal with tasks that are waiting result.
while (!QUEUE_EMPTY(&gUvProcTaskQueue)) {
QUEUE* h = QUEUE_HEAD(&gUvProcTaskQueue);
while (!QUEUE_EMPTY(&udfc->gUvProcTaskQueue)) {
QUEUE* h = QUEUE_HEAD(&udfc->gUvProcTaskQueue);
QUEUE_REMOVE(h);
SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue);
if (gUdfcState == UDFC_STATE_STOPPING) {
if (udfc->gUdfcState == UDFC_STATE_STOPPING) {
task->errCode = UDFC_CODE_STOPPING;
} else if (gUdfcState == UDFC_STATE_RESTARTING) {
task->errCode = UDFC_CODE_RESTARTING;
}
uv_sem_post(&task->taskSem);
}
}
void udfStopAsyncCb(uv_async_t *async) {
cleanUpUvTasks();
if (gUdfcState == UDFC_STATE_STOPPING) {
uv_stop(&gUdfdLoop);
SUdfdProxy *udfc = async->data;
cleanUpUvTasks(udfc);
if (udfc->gUdfcState == UDFC_STATE_STOPPING) {
uv_stop(&udfc->gUdfdLoop);
}
}
int32_t udfcSpawnUdfd();
void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) {
//TODO: pipe close will be first received
debugPrint("Process exited with status %" PRId64 ", signal %d", exit_status, term_signal);
uv_close((uv_handle_t *) req, NULL);
//TODO: restart the udfd process
if (gUdfcState == UDFC_STATE_STOPPING) {
if (term_signal != SIGINT) {
//TODO: log error
}
}
if (gUdfcState == UDFC_STATE_READY) {
gUdfcState = UDFC_STATE_RESTARTING;
//TODO: asynchronous without blocking. how to do it
//cleanUpUvTasks();
udfcSpawnUdfd();
}
}
int32_t udfcSpawnUdfd() {
//TODO: path
uv_process_options_t options = {0};
static char path[256] = {0};
size_t cwdSize;
uv_cwd(path, &cwdSize);
strcat(path, "/udfd");
char* args[2] = {path, NULL};
options.args = args;
options.file = path;
options.exit_cb = onUdfdExit;
options.stdio_count = 3;
uv_stdio_container_t child_stdio[3];
child_stdio[0].flags = UV_IGNORE;
child_stdio[1].flags = UV_INHERIT_FD;
child_stdio[1].data.fd = 1;
child_stdio[2].flags = UV_INHERIT_FD;
child_stdio[2].data.fd = 2;
options.stdio = child_stdio;
//TODO spawn error
int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options);
if (err != 0) {
debugPrint("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err));
}
return err;
}
void constructUdfService(void *argsThread) {
uv_loop_init(&gUdfdLoop);
uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb);
uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb);
uv_mutex_init(&gUdfTaskQueueMutex);
QUEUE_INIT(&gUdfTaskQueue);
QUEUE_INIT(&gUvProcTaskQueue);
uv_barrier_wait(&gUdfInitBarrier);
SUdfdProxy *udfc = (SUdfdProxy*)argsThread;
uv_loop_init(&udfc->gUdfdLoop);
uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopTaskAync, udfClientAsyncCb);
udfc->gUdfLoopTaskAync.data = udfc;
uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopStopAsync, udfStopAsyncCb);
udfc->gUdfLoopStopAsync.data = udfc;
uv_mutex_init(&udfc->gUdfTaskQueueMutex);
QUEUE_INIT(&udfc->gUdfTaskQueue);
QUEUE_INIT(&udfc->gUvProcTaskQueue);
uv_barrier_wait(&udfc->gUdfInitBarrier);
//TODO return value of uv_run
uv_run(&gUdfdLoop, UV_RUN_DEFAULT);
uv_loop_close(&gUdfdLoop);
uv_run(&udfc->gUdfdLoop, UV_RUN_DEFAULT);
uv_loop_close(&udfc->gUdfdLoop);
}
int32_t createUdfdProxy(int32_t dnodeId) {
gUdfcState = UDFC_STATE_STARTNG;
uv_barrier_init(&gUdfInitBarrier, 2);
uv_thread_create(&gUdfLoopThread, constructUdfService, 0);
uv_barrier_wait(&gUdfInitBarrier); gUdfcState = UDFC_STATE_READY;
int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) {
SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy));
proxy->dnodeId = dnodeId;
proxy->gUdfcState = UDFC_STATE_STARTNG;
uv_barrier_init(&proxy->gUdfInitBarrier, 2);
uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy);
uv_barrier_wait(&proxy->gUdfInitBarrier);
proxy->gUdfcState = UDFC_STATE_READY;
*udfc = proxy;
return 0;
}
int32_t destroyUdfdProxy(int32_t dnodeId) {
gUdfcState = UDFC_STATE_STOPPING;
uv_barrier_destroy(&gUdfInitBarrier);
// if (gUdfcState == UDFC_STATE_STOPPING) {
// uv_process_kill(&gUdfdProcess, SIGINT);
// }
uv_async_send(&gUdfLoopStopAsync);
uv_thread_join(&gUdfLoopThread);
uv_mutex_destroy(&gUdfTaskQueueMutex);
gUdfcState = UDFC_STATUS_FINAL;
int32_t udfcClose(UdfcHandle udfcHandle) {
SUdfdProxy *udfc = udfcHandle;
udfc->gUdfcState = UDFC_STATE_STOPPING;
uv_async_send(&udfc->gUdfLoopStopAsync);
uv_thread_join(&udfc->gUdfLoopThread);
uv_mutex_destroy(&udfc->gUdfTaskQueueMutex);
uv_barrier_destroy(&udfc->gUdfInitBarrier);
udfc->gUdfcState = UDFC_STATUS_FINAL;
taosMemoryFree(udfc);
return 0;
}
......@@ -1026,11 +980,12 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) {
return task->errCode;
}
int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle) {
int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) {
debugPrint("%s", "client setup udf");
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
task->session = taosMemoryMalloc(sizeof(SUdfUvSession));
task->session->udfc = udfc;
task->type = UDF_TASK_SETUP;
SUdfSetupRequest *req = &task->_setup.req;
......@@ -1046,13 +1001,13 @@ int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle) {
SUdfSetupResponse *rsp = &task->_setup.rsp;
task->session->severHandle = rsp->udfHandle;
*handle = task->session;
*funcHandle = task->session;
int32_t err = task->errCode;
taosMemoryFree(task);
return err;
}
int32_t callUdf(UdfHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
SSDataBlock* output, SUdfInterBuf *newState) {
debugPrint("%s", "client call udf");
......@@ -1121,7 +1076,7 @@ int32_t callUdf(UdfHandle handle, int8_t callType, SSDataBlock *input, SUdfInter
}
//TODO: translate these calls to callUdf
int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf) {
int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) {
int8_t callType = TSDB_UDF_CALL_AGG_INIT;
int32_t err = callUdf(handle, callType, NULL, NULL, NULL, NULL, interBuf);
......@@ -1131,7 +1086,7 @@ int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf) {
// input: block, state
// output: interbuf,
int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) {
int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) {
int8_t callType = TSDB_UDF_CALL_AGG_PROC;
int32_t err = callUdf(handle, callType, block, state, NULL, NULL, newState);
return err;
......@@ -1139,7 +1094,7 @@ int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *st
// input: interbuf1, interbuf2
// output: resultBuf
int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) {
int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) {
int8_t callType = TSDB_UDF_CALL_AGG_MERGE;
int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf);
return err;
......@@ -1147,7 +1102,7 @@ int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf
// input: interBuf
// output: resultData
int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) {
int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) {
int8_t callType = TSDB_UDF_CALL_AGG_PROC;
int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData);
return err;
......@@ -1155,13 +1110,13 @@ int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBu
// input: block
// output: resultData
int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData) {
int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData) {
int8_t callType = TSDB_UDF_CALL_SCALA_PROC;
int32_t err = callUdf(handle, callType, block, NULL, NULL, resultData, NULL);
return err;
}
int32_t teardownUdf(UdfHandle handle) {
int32_t teardownUdf(UdfcFuncHandle handle) {
debugPrint("%s", "client teardown udf");
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
......
source/libs/function/src/udfd.c
浏览文件 @ 312b7a4f
......@@ -27,7 +27,10 @@
typedef struct SUdfdContext {
uv_loop_t *loop;
uv_pipe_t ctrlPipe;
uv_signal_t intrSignal;
char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN];
uv_pipe_t listeningPipe;
void *clientRpc;
uv_mutex_t udfsMutex;
......@@ -76,9 +79,9 @@ typedef struct SUdf {
// TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
// TODO: add private udf structure.
typedef struct SUdfHandle {
typedef struct SUdfcFuncHandle {
SUdf *udf;
} SUdfHandle;
} SUdfcFuncHandle;
int32_t udfdLoadUdf(char* udfName, SUdf* udf) {
strcpy(udf->name, udfName);
......@@ -143,7 +146,7 @@ void udfdProcessRequest(uv_work_t *req) {
}
uv_mutex_unlock(&udf->lock);
}
SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle));
SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle));
handle->udf = udf;
// TODO: allocate private structure and call init function and set it to handle
SUdfResponse rsp;
......@@ -166,7 +169,7 @@ void udfdProcessRequest(uv_work_t *req) {
case UDF_TASK_CALL: {
SUdfCallRequest *call = &request.call;
fnDebug("%"PRId64 "call request. call type %d, handle: %"PRIx64, request.seqNum, call->callType, call->udfHandle);
SUdfHandle *handle = (SUdfHandle *)(call->udfHandle);
SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle);
SUdf *udf = handle->udf;
SUdfDataBlock input = {0};
......@@ -204,7 +207,7 @@ void udfdProcessRequest(uv_work_t *req) {
case UDF_TASK_TEARDOWN: {
SUdfTeardownRequest *teardown = &request.teardown;
fnInfo("teardown. %"PRId64"handle:%"PRIx64, request.seqNum, teardown->udfHandle)
SUdfHandle *handle = (SUdfHandle *)(teardown->udfHandle);
SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle);
SUdf *udf = handle->udf;
bool unloadUdf = false;
uv_mutex_lock(&global.udfsMutex);
......@@ -380,10 +383,12 @@ void udfdOnNewConnection(uv_stream_t *server, int status) {
}
}
void removeListeningPipe(int sig) {
void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
fnInfo("udfd signal received: %d\n", signum);
uv_fs_t req;
uv_fs_unlink(global.loop, &req, "udf.sock", NULL);
exit(0);
uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
uv_signal_stop(handle);
uv_stop(global.loop);
}
void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
......@@ -492,37 +497,67 @@ static int32_t udfdInitLog() {
return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
}
void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) {
buf->base = taosMemoryMalloc(suggested_size);
buf->len = suggested_size;
}
void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) {
if (nread < 0) {
fnError("udfd ctrl pipe read error. %s", uv_err_name(nread));
uv_close((uv_handle_t*)q, NULL);
uv_stop(global.loop);
return;
}
fnError("udfd ctrl pipe read %zu bytes", nread);
taosMemoryFree(buf->base);
}
static int32_t removeListeningPipe() {
uv_fs_t req;
int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
uv_fs_req_cleanup(&req);
return err;
}
static int32_t udfdUvInit() {
uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t));
if (loop) {
uv_loop_init(loop);
}
global.loop = loop;
uv_pipe_init(global.loop, &global.ctrlPipe, 1);
uv_pipe_open(&global.ctrlPipe, 0);
uv_read_start((uv_stream_t*)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
char dnodeId[8] = {0};
size_t dnodeIdSize;
uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
if (err != 0) {
dnodeId[0] = '1';
}
char listenPipeName[32] = {0};
snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
strcpy(global.listenPipeName, listenPipeName);
uv_fs_t req;
uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
removeListeningPipe();
uv_pipe_t server;
uv_pipe_init(global.loop, &server, 0);
uv_pipe_init(global.loop, &global.listeningPipe, 0);
signal(SIGINT, removeListeningPipe);
uv_signal_init(global.loop, &global.intrSignal);
uv_signal_start(&global.intrSignal, udfdIntrSignalHandler, SIGINT);
int r;
fnInfo("bind to pipe %s", global.listenPipeName);
if ((r = uv_pipe_bind(&server, listenPipeName))) {
if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) {
fnError("Bind error %s", uv_err_name(r));
removeListeningPipe(0);
removeListeningPipe();
return -1;
}
if ((r = uv_listen((uv_stream_t *)&server, 128, udfdOnNewConnection))) {
if ((r = uv_listen((uv_stream_t *)&global.listeningPipe, 128, udfdOnNewConnection))) {
fnError("Listen error %s", uv_err_name(r));
removeListeningPipe(0);
removeListeningPipe();
return -2;
}
return 0;
......
source/libs/function/test/runUdf.c
浏览文件 @ 312b7a4f
......@@ -8,7 +8,8 @@
#include "tdatablock.h"
int main(int argc, char *argv[]) {
createUdfdProxy(1);
UdfcHandle udfc;
udfcOpen(1, &udfc);
uv_sleep(1000);
char path[256] = {0};
size_t cwdSize = 256;
......@@ -20,9 +21,9 @@ int main(int argc, char *argv[]) {
fprintf(stdout, "current working directory:%s\n", path);
strcat(path, "/libudf1.so");
UdfHandle handle;
UdfcFuncHandle handle;
SEpSet epSet;
setupUdf("udf1", &epSet, &handle);
setupUdf(udfc, "udf1", &epSet, &handle);
SSDataBlock block = {0};
SSDataBlock* pBlock = &block;
......@@ -53,5 +54,5 @@ int main(int argc, char *argv[]) {
}
teardownUdf(handle);
destroyUdfdProxy(1);
udfcClose(udfc);
}
source/libs/parser/src/parTranslater.c
浏览文件 @ 312b7a4f
......@@ -585,7 +585,8 @@ static EDealRes haveAggFunction(SNode* pNode, void* pContext) {
}
static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet};
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(&param, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
}
pCxt->errCode = fmGetFuncResultType(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len);
......@@ -1918,7 +1919,8 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION);
}
SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0);
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet};
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(&param, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
}
return TSDB_CODE_SUCCESS;
......
source/libs/planner/src/planLogicCreater.c
浏览文件 @ 312b7a4f
......@@ -85,13 +85,19 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
static int32_t rewriteExprForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
nodesWalkExprs(pExprs, doNameExpr, NULL);
SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
return cxt.errCode;
}
static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) {
SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
nodesRewriteExprs(pTarget, doRewriteExpr, &cxt);
return cxt.errCode;
}
static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) {
if (NULL == pNewRoot->pChildren) {
pNewRoot->pChildren = nodesMakeList();
......@@ -433,10 +439,10 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
code = rewriteExprForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) {
......@@ -472,7 +478,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
code = rewriteExprForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
}
if (TSDB_CODE_SUCCESS == code) {
......@@ -723,7 +729,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT);
code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT);
}
// set the output
......@@ -850,6 +856,37 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator
return code;
}
static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) {
SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG);
if (NULL == pAgg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = TSDB_CODE_SUCCESS;
pAgg->pGroupKeys = nodesCloneList(pSetOperator->pProjectionList);
if (NULL == pAgg->pGroupKeys) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprs(pAgg->pGroupKeys, pSetOperator->pOrderByList);
}
// set the output
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pAgg;
} else {
nodesDestroyNode(pAgg);
}
return code;
}
static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) {
SLogicNode* pSetOp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
......@@ -857,6 +894,9 @@ static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetO
case SET_OP_TYPE_UNION_ALL:
code = createSetOpProjectLogicNode(pCxt, pSetOperator, &pSetOp);
break;
case SET_OP_TYPE_UNION:
code = createSetOpAggLogicNode(pCxt, pSetOperator, &pSetOp);
break;
default:
code = -1;
break;
......
source/libs/planner/src/planOptimizer.c
浏览文件 @ 312b7a4f
......@@ -228,6 +228,8 @@ static int32_t cpdMergeConds(SNode** pDst, SNodeList** pSrc) {
if (NULL == pLogicCond) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
pLogicCond->condType = LOGIC_COND_TYPE_AND;
pLogicCond->pParameterList = *pSrc;
*pDst = (SNode*)pLogicCond;
......
source/libs/planner/src/planSpliter.c
浏览文件 @ 312b7a4f
......@@ -50,6 +50,11 @@ typedef struct SUaInfo {
SLogicSubplan* pSubplan;
} SUaInfo;
typedef struct SUnInfo {
SAggLogicNode* pAgg;
SLogicSubplan* pSubplan;
} SUnInfo;
typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, void* pInfo);
static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan, int32_t flag) {
......@@ -226,7 +231,6 @@ static SLogicSubplan* uaCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) {
pSubplan->id.groupId = pCxt->groupId;
pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
pSubplan->pNode = pNode;
// TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo*);
return pSubplan;
}
......@@ -244,24 +248,22 @@ static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan
pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
return nodesListMakeAppend(&pProject->node.pChildren, (SNode*)pExchange);
// if (NULL == pProject->node.pParent) {
// pSubplan->pNode = (SLogicNode*)pExchange;
// nodesDestroyNode(pProject);
// return TSDB_CODE_SUCCESS;
// }
// SNode* pNode;
// FOREACH(pNode, pProject->node.pParent->pChildren) {
// if (nodesEqualNode(pNode, pProject)) {
// REPLACE_NODE(pExchange);
// nodesDestroyNode(pNode);
// return TSDB_CODE_SUCCESS;
// }
// }
// nodesDestroyNode(pExchange);
// return TSDB_CODE_FAILED;
if (NULL == pProject->node.pParent) {
pSubplan->pNode = (SLogicNode*)pExchange;
nodesDestroyNode(pProject);
return TSDB_CODE_SUCCESS;
}
SNode* pNode;
FOREACH(pNode, pProject->node.pParent->pChildren) {
if (nodesEqualNode(pNode, pProject)) {
REPLACE_NODE(pExchange);
nodesDestroyNode(pNode);
return TSDB_CODE_SUCCESS;
}
}
nodesDestroyNode(pExchange);
return TSDB_CODE_FAILED;
}
static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
......@@ -291,10 +293,78 @@ static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
return code;
}
static SLogicNode* unMatchByNode(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) {
return pNode;
}
SNode* pChild;
FOREACH(pChild, pNode->pChildren) {
SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild);
if (NULL != pSplitNode) {
return pSplitNode;
}
}
return NULL;
}
static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SAggLogicNode* pAgg) {
SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE);
if (NULL == pExchange) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pExchange->srcGroupId = pCxt->groupId;
// pExchange->precision = pScan->pMeta->tableInfo.precision;
pExchange->node.pTargets = nodesCloneList(pAgg->node.pTargets);
if (NULL == pExchange->node.pTargets) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
return nodesListMakeAppend(&pAgg->node.pChildren, pExchange);
}
static bool unFindSplitNode(SLogicSubplan* pSubplan, SUnInfo* pInfo) {
SLogicNode* pSplitNode = unMatchByNode(pSubplan->pNode);
if (NULL != pSplitNode) {
pInfo->pAgg = (SAggLogicNode*)pSplitNode;
pInfo->pSubplan = pSubplan;
}
return NULL != pSplitNode;
}
static int32_t unSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
SUnInfo info = {0};
if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unFindSplitNode, &info)) {
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
SNode* pChild = NULL;
FOREACH(pChild, info.pAgg->node.pChildren) {
code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild));
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
} else {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
nodesClearList(info.pAgg->node.pChildren);
info.pAgg->node.pChildren = NULL;
code = unCreateExchangeNode(pCxt, info.pSubplan, info.pAgg);
}
++(pCxt->groupId);
pCxt->split = true;
return code;
}
static const SSplitRule splitRuleSet[] = {
{ .pName = "SuperTableScan", .splitFunc = stsSplit },
{ .pName = "ChildTableJoin", .splitFunc = ctjSplit },
{ .pName = "UnionAll", .splitFunc = uaSplit },
{ .pName = "Union", .splitFunc = unSplit }
};
static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule));
......
source/libs/planner/test/planSetOpTest.cpp
浏览文件 @ 312b7a4f
......@@ -27,3 +27,9 @@ TEST_F(PlanSetOpTest, unionAll) {
run("select c1, c2 from t1 where c1 > 10 union all select c1, c2 from t1 where c1 > 20");
}
TEST_F(PlanSetOpTest, union) {
useDb("root", "test");
run("select c1, c2 from t1 where c1 > 10 union select c1, c2 from t1 where c1 > 20");
}
source/libs/planner/test/planTestUtil.cpp
浏览文件 @ 312b7a4f
......@@ -62,7 +62,7 @@ public:
doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan);
SQueryPlan* pPlan = nullptr;
doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan, NULL);
doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan);
if (g_isDump) {
dump();
......@@ -162,7 +162,8 @@ private:
res_.scaledLogicPlan_ = toString((SNode*)(*pLogicPlan));
}
void doCreatePhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) {
void doCreatePhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) {
SArray* pExecNodeList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr));
DO_WITH_THROW(createPhysiPlan, pCxt, pLogicPlan, pPlan, pExecNodeList);
res_.physiPlan_ = toString((SNode*)(*pPlan));
SNode* pNode;
......
source/libs/qcom/src/querymsg.c
浏览文件 @ 312b7a4f
......@@ -157,6 +157,28 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t
return TSDB_CODE_SUCCESS;
}
int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
SRetrieveFuncReq funcReq = {0};
funcReq.numOfFuncs = 1;
funcReq.ignoreCodeComment = true;
funcReq.pFuncNames = taosArrayInit(1, strlen(input) + 1);
taosArrayPush(funcReq.pFuncNames, input);
int32_t bufLen = tSerializeSRetrieveFuncReq(NULL, 0, &funcReq);
void *pBuf = rpcMallocCont(bufLen);
tSerializeSRetrieveFuncReq(pBuf, bufLen, &funcReq);
taosArrayDestroy(funcReq.pFuncNames);
*msg = pBuf;
*msgLen = bufLen;
return TSDB_CODE_SUCCESS;
}
int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) {
SUseDbOutput *pOut = output;
......@@ -379,6 +401,31 @@ int32_t queryProcessGetIndexRsp(void *output, char *msg, int32_t msgSize) {
return TSDB_CODE_SUCCESS;
}
int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) {
SRetrieveFuncRsp out = {0};
if (NULL == output || NULL == msg || msgSize <= 0) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
if (tDeserializeSRetrieveFuncRsp(msg, msgSize, &out) != 0) {
qError("tDeserializeSRetrieveFuncRsp failed, msgSize:%d", msgSize);
return TSDB_CODE_INVALID_MSG;
}
if (1 != out.numOfFuncs) {
qError("invalid func num returned, numOfFuncs:%d", out.numOfFuncs);
return TSDB_CODE_INVALID_MSG;
}
SFuncInfo * funcInfo = taosArrayGet(out.pFuncInfos, 0);
memcpy(output, funcInfo, sizeof(*funcInfo));
taosArrayDestroy(out.pFuncInfos);
return TSDB_CODE_SUCCESS;
}
void initQueryModuleMsgHandle() {
queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg;
queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg;
......@@ -386,6 +433,7 @@ void initQueryModuleMsgHandle() {
queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryBuildQnodeListMsg;
queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg;
queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg;
queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg;
queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp;
queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp;
......@@ -393,6 +441,7 @@ void initQueryModuleMsgHandle() {
queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryProcessQnodeListRsp;
queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp;
queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp;
queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp;
}
#pragma GCC diagnostic pop
source/libs/transport/inc/transportInt.h
浏览文件 @ 312b7a4f
......@@ -63,6 +63,7 @@ typedef struct {
void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey);
int (*retry)(void* parent, SRpcMsg*, SEpSet*);
int32_t refCount;
void* parent;
......
source/libs/transport/src/trans.c
浏览文件 @ 312b7a4f
......@@ -39,6 +39,7 @@ void* rpcOpen(const SRpcInit* pInit) {
// register callback handle
pRpc->cfp = pInit->cfp;
pRpc->afp = pInit->afp;
pRpc->retry = pInit->rfp;
if (pInit->connType == TAOS_CONN_SERVER) {
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
......@@ -100,11 +101,10 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) {
SRpcMsg rpcMsg;
memset(&rpcMsg, 0, sizeof(rpcMsg));
rpcMsg.contLen = sizeof(SEpSet);
rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
if (rpcMsg.pCont == NULL) return;
memcpy(rpcMsg.pCont, pEpSet, sizeof(SEpSet));
SMEpSet msg = {.epSet = *pEpSet};
int32_t len = tSerializeSMEpSet(NULL, 0, &msg);
rpcMsg.pCont = rpcMallocCont(len);
tSerializeSMEpSet(rpcMsg.pCont, len, &msg);
rpcMsg.code = TSDB_CODE_RPC_REDIRECT;
rpcMsg.handle = thandle;
......
source/libs/transport/src/transCli.c
浏览文件 @ 312b7a4f
......@@ -33,10 +33,6 @@ typedef struct SCliConn {
bool broken; // link broken or not
ConnStatus status; //
int release; // 1: release
// spi configure
char spi;
char secured;
char* ip;
uint32_t port;
......@@ -44,7 +40,6 @@ typedef struct SCliConn {
// debug and log info
struct sockaddr_in addr;
struct sockaddr_in locaddr;
} SCliConn;
typedef struct SCliMsg {
......@@ -102,6 +97,8 @@ static void cliSendCb(uv_write_t* req, int status);
static void cliConnCb(uv_connect_t* req, int status);
static void cliAsyncCb(uv_async_t* handle);
static void cliAppCb(SCliConn* pConn, STransMsg* pMsg);
static SCliConn* cliCreateConn(SCliThrdObj* thrd);
static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/);
static void cliDestroy(uv_handle_t* handle);
......@@ -303,8 +300,6 @@ void cliHandleResp(SCliConn* conn) {
TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port),
taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen);
conn->secured = pHead->secured;
if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) {
tTrace("except, server continue send while cli ignore it");
// transUnrefCliHandle(conn);
......@@ -318,7 +313,8 @@ void cliHandleResp(SCliConn* conn) {
if (pCtx == NULL || pCtx->pSem == NULL) {
tTrace("%s cli conn %p handle resp", pTransInst->label, conn);
(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
cliAppCb(conn, &transMsg);
//(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
} else {
tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, conn);
memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg));
......@@ -384,7 +380,8 @@ void cliHandleExcept(SCliConn* pConn) {
once = true;
continue;
}
(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
cliAppCb(pConn, &transMsg);
//(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
} else {
tTrace("%s cli conn(sync) %p handle except", pTransInst->label, pConn);
memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg));
......@@ -884,6 +881,18 @@ int cliRBChoseIdx(STrans* pTransInst) {
}
return index % pTransInst->numOfThreads;
}
void cliAppCb(SCliConn* pConn, STransMsg* transMsg) {
SCliThrdObj* pThrd = pConn->hostThrd;
STrans* pTransInst = pThrd->pTransInst;
if (transMsg->code == TSDB_CODE_RPC_REDIRECT && pTransInst->retry != NULL) {
SMEpSet emsg = {0};
tDeserializeSMEpSet(transMsg->pCont, transMsg->contLen, &emsg);
pTransInst->retry(pTransInst, transMsg, &(emsg.epSet));
} else {
pTransInst->cfp(pTransInst->parent, transMsg, NULL);
}
}
void transCloseClient(void* arg) {
SCliObj* cli = arg;
......
source/libs/transport/src/transSrv.c
浏览文件 @ 312b7a4f
......@@ -30,7 +30,6 @@ typedef struct SSrvConn {
uv_timer_t pTimer;
queue queue;
int ref;
int persist; // persist connection or not
SConnBuffer readBuf; // read buf,
int inType;
......@@ -692,8 +691,6 @@ static void uvDestroyConn(uv_handle_t* handle) {
if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) {
tTrace("work thread quit");
uv_walk(thrd->loop, uvWalkCb, NULL);
// uv_loop_close(thrd->loop);
// uv_stop(thrd->loop);
}
}
......@@ -756,8 +753,6 @@ void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) {
thrd->quit = true;
if (QUEUE_IS_EMPTY(&thrd->conn)) {
uv_walk(thrd->loop, uvWalkCb, NULL);
// uv_loop_close(thrd->loop);
// uv_stop(thrd->loop);
} else {
destroyAllConn(thrd);
}
......@@ -851,10 +846,8 @@ void transRefSrvHandle(void* handle) {
if (handle == NULL) {
return;
}
SSrvConn* conn = handle;
int ref = T_REF_INC((SSrvConn*)handle);
UNUSED(ref);
tDebug("server conn %p ref count: %d", handle, ref);
}
void transUnrefSrvHandle(void* handle) {
......
source/os/src/osEnv.c
浏览文件 @ 312b7a4f
......@@ -37,6 +37,7 @@ int64_t tsOpenMax = 0;
int64_t tsStreamMax = 0;
float tsNumOfCores = 0;
int64_t tsTotalMemoryKB = 0;
char* tsProcPath = NULL;
void osDefaultInit() {
taosSeedRand(taosSafeRand());
......
source/os/src/osFile.c
浏览文件 @ 312b7a4f
......@@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
#if FILE_WITH_LOCK
taosThreadRwlockWrlock(&(pFile->rwlock));
#endif
/*assert(pFile->fd >= 0); // Please check if you have closed the file.*/
assert(pFile->fd >= 0); // Please check if you have closed the file.
int64_t nleft = count;
int64_t nwritten = 0;
......
source/os/src/osProc.c
浏览文件 @ 312b7a4f
......@@ -17,7 +17,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
static char *tsProcPath = NULL;
char *tsProcPath = NULL;
int32_t taosNewProc(char **args) {
int32_t pid = fork();
......
source/util/src/talgo.c
浏览文件 @ 312b7a4f
......@@ -229,22 +229,21 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
}
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) {
__ext_compar_fn_t compar, char* buf, bool maxroot) {
int32_t parent;
int32_t child;
char *buf;
char* tmp = NULL;
if (buf == NULL) {
tmp = taosMemoryMalloc(size);
} else {
tmp = buf;
}
if (base && size > 0 && compar) {
parent = start;
child = 2 * parent + 1;
if (swap == NULL) {
buf = taosMemoryCalloc(1, size);
if (buf == NULL) {
return;
}
}
if (maxroot) {
while (child <= end) {
if (child + 1 <= end &&
......@@ -256,11 +255,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
break;
}
if (swap == NULL) {
doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf);
} else {
(*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap);
}
doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, tmp);
parent = child;
child = 2 * parent + 1;
......@@ -276,33 +271,35 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
break;
}
if (swap == NULL) {
doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf);
} else {
(*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap);
}
doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, tmp);
parent = child;
child = 2 * parent + 1;
}
}
if (swap == NULL) {
taosMemoryFreeClear(buf);
}
if (buf == NULL) {
taosMemoryFree(tmp);
}
}
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
const void *parswap, __ext_swap_fn_t swap, bool maxroot) {
bool maxroot) {
int32_t i;
char* buf = taosMemoryCalloc(1, size);
if (buf == NULL) {
return;
}
if (base && size > 0) {
for (i = len / 2 - 1; i >= 0; i--) {
taosheapadjust(base, size, i, len - 1, parcompar, compar, parswap, swap, maxroot);
taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot);
}
}
taosMemoryFree(buf);
/*
char *buf = taosMemoryCalloc(1, size);
......
source/util/src/tlog.c
浏览文件 @ 312b7a4f
......@@ -222,7 +222,7 @@ static void *taosThreadToOpenNewFile(void *param) {
tsLogObj.logHandle->pFile = pFile;
tsLogObj.lines = 0;
tsLogObj.openInProgress = 0;
taosSsleep(3);
taosSsleep(10);
taosCloseLogByFd(pOldFile);
uInfo(" new log file:%d is opened", tsLogObj.flag);
......
tests/script/test.sh
浏览文件 @ 312b7a4f
......@@ -131,8 +131,8 @@ if [ -n "$FILE_NAME" ]; then
FLAG="-v"
fi
echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
else
if [[ $MULTIPROCESS -eq 1 ]];then
echo "ExcuteCmd(multiprocess):" $PROGRAM -m -c $CFG_DIR -f $FILE_NAME
......
tests/script/tsim/query/session.sim
浏览文件 @ 312b7a4f
......@@ -279,7 +279,8 @@ endi
print ================> syntax error check not active ================> reactive
sql_error select * from dev_001 session(ts,1w)
sql_error select count(*) from st session(ts,1w)
print disable this temporarily, session can not be directly applied to super table.
#sql_error select count(*) from st session(ts,1w)
sql_error select count(*) from dev_001 group by tagtype session(ts,1w)
sql_error sql select count(*) from dev_001 session(ts,1n)
sql_error sql select count(*) from dev_001 session(ts,1y)
......
tests/script/tsim/tmq/basic1.sim
浏览文件 @ 312b7a4f
......@@ -8,196 +8,245 @@
#
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
#
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
run tsim/tmq/prepareBasicEnv.sim
#---- global parameters start ----#
$dbName = db
$vgroups = 1
$stbPrefix = stb
$ctbPrefix = ctb
$ntbPrefix = ntb
$stbNum = 1
$ctbNum = 10
$ntbNum = 10
$rowsPerCtb = 100
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----#
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
$pullDelay = 3
$ifcheckdata = 1
$showMsg = 1
$showRow = 0
sql connect
sql use $dbName
$dbNamme = d0
print =============== create database , vgroup 4
sql create database $dbNamme vgroups 4
sql use $dbNamme
print =============== create super table
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
sql show stables
if $rows != 1 then
return -1
endi
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print =============== create child table
sql create table ct0 using stb tags(1000)
sql create table ct1 using stb tags(2000)
#sql create table ct3 using stb tags(3000)
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print =============== create normal table
sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
print =============== create multi topics. notes: now only support:
print =============== 1. columns from stb; 2. * from ctb; 3. columns from ctb
print =============== will support: * from stb; function from stb/ctb
#sql show topics
#if $rows != 9 then
# return -1
#endi
sql create topic topic_stb_column as select ts, c1, c3 from stb
#sql create topic topic_stb_all as select * from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
$keyList = ' . group.id:cgrp1
$keyList = $keyList . '
sql create topic topic_ctb_column as select ts, c1, c3 from ct0
sql create topic topic_ctb_all as select * from ct0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
print ================ test consume from stb
$loop_cnt = 0
loop_consume_diff_topic_from_stb:
if $loop_cnt == 0 then
print == scenario 1: topic_stb_column
$topicList = ' . topic_stb_column
$topicList = $topicList . '
elif $loop_cnt == 1 then
print == scenario 2: topic_stb_all
$topicList = ' . topic_stb_all
$topicList = $topicList . '
elif $loop_cnt == 2 then
print == scenario 3: topic_stb_function
$topicList = ' . topic_stb_function
$topicList = $topicList . '
else
goto loop_consume_diff_topic_from_stb_end
endi
sql create topic topic_ntb_column as select ts, c1, c3 from ntb
sql create topic topic_ntb_all as select * from ntb
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
$consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb
#$expectmsgcnt = $totalMsgOfStb + 1
$expectmsgcnt = 110
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
print == check consume result
wait_consumer_end_from_stb:
sql select * from consumeresult
print ==> rows: $rows
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $rows != 1 then
sleep 1000
goto wait_consumer_end_from_stb
endi
if $data[0][1] != $consumerId then
return -1
endi
if $data[0][2] != $expectmsgcnt then
return -1
endi
if $data[0][3] != $expectmsgcnt then
return -1
endi
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_stb
loop_consume_diff_topic_from_stb_end:
#######################################################################################
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb1
sql create database $cdbName vgroups 1
sleep 500
sql use $cdbName
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables
if $rows != 3 then
if $rows != 2 then
return -1
endi
#######################################################################################
print =============== insert data
$tbPrefix = ct
$tbNum = 2
$rowNum = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
print ================ test consume from ctb
$loop_cnt = 0
loop_consume_diff_topic_from_ctb:
if $loop_cnt == 0 then
print == scenario 1: topic_ctb_column
$topicList = ' . topic_ctb_column
$topicList = $topicList . '
elif $loop_cnt == 1 then
print == scenario 2: topic_ctb_all
$topicList = ' . topic_ctb_all
$topicList = $topicList . '
elif $loop_cnt == 2 then
print == scenario 3: topic_ctb_function
$topicList = ' . topic_ctb_function
$topicList = $topicList . '
else
goto loop_consume_diff_topic_from_ctb_end
endi
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
$x = 0
while $x < $rowNum
$c = $x / 10
$c = $c * 10
$c = $x - $c
$binary = ' . binary
$binary = $binary . $c
$binary = $binary . '
sql insert into $tb values ($tstart , $c , $x , $binary )
sql insert into ntb values ($tstart , $c , $x , $binary )
$tstart = $tstart + 1
$x = $x + 1
endw
$i = $i + 1
$tstart = 1640966400000
endw
#root@trd02 /home $ tmq_sim --help
# -c Configuration directory, default is
# -d The name of the database for cosumer, no default
# -t The topic string for cosumer, no default
# -k The key-value string for cosumer, no default
# -g showMsgFlag, default is 0
#
$consumerId = 0
$totalMsgOfCtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfCtb + 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
$totalMsgCnt = $rowNum * $tbNum
print inserted totalMsgCnt: $totalMsgCnt
# supported key:
# group.id:<xxx>
# enable.auto.commit:<true | false>
# auto.offset.reset:<earliest | latest | none>
# td.connect.ip:<fqdn | ipaddress>
# td.connect.user:root
# td.connect.pass:taosdata
# td.connect.port:6030
# td.connect.db:db
system_content echo -n \$BUILD_DIR
$tmq_sim = $system_content . /build/bin/tmq_sim
system_content echo -n \$SIM_DIR
$tsim_cfg = $system_content . /tsim/cfg
print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 20, 0}@ then
print == check consume result
wait_consumer_end_from_ctb:
sql select * from consumeresult
print ==> rows: $rows
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $rows != 1 then
sleep 1000
goto wait_consumer_end_from_ctb
endi
if $data[0][1] != $consumerId then
return -1
endi
#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
#print cmd result----> $system_content
#if $system_content != @{consume success: 20, 0}@ then
# return -1
#endi
print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2"
system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 20, 0}@ then
print expect @{consume success: 20, 0}@, actual: $system_content
if $data[0][2] != $totalMsgOfCtb then
return -1
endi
print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 10, 0}@ then
if $data[0][3] != $totalMsgOfCtb then
return -1
endi
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ctb
loop_consume_diff_topic_from_ctb_end:
#######################################################################################
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb2
sql create database $cdbName vgroups 1
sleep 500
sql use $cdbName
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 10, 0}@ then
sql show tables
if $rows != 2 then
return -1
endi
#######################################################################################
print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2"
system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 10, 0}@ then
return -1
print ================ test consume from ntb
$loop_cnt = 0
loop_consume_diff_topic_from_ntb:
if $loop_cnt == 0 then
print == scenario 1: topic_ntb_column
$topicList = ' . topic_ntb_column
$topicList = $topicList . '
elif $loop_cnt == 1 then
print == scenario 2: topic_ntb_all
$topicList = ' . topic_ntb_all
$topicList = $topicList . '
elif $loop_cnt == 2 then
print == scenario 3: topic_ntb_function
$topicList = ' . topic_ntb_function
$topicList = $topicList . '
else
goto loop_consume_diff_topic_from_ntb_end
endi
print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 20, 0}@ then
$consumerId = 0
$totalMsgOfNtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfNtb + 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
print == check consume result from ntb
wait_consumer_end_from_ntb:
sql select * from consumeresult
print ==> rows: $rows
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
if $rows != 1 then
sleep 1000
goto wait_consumer_end_from_ntb
endi
if $data[0][1] != $consumerId then
return -1
endi
print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 20, 0}@ then
if $data[0][2] != $totalMsgOfNtb then
return -1
endi
print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2"
system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 20, 0}@ then
if $data[0][3] != $totalMsgOfNtb then
return -1
endi
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ntb
loop_consume_diff_topic_from_ntb_end:
print =============== create database , vgroup 4
$dbNamme = d1
sql create database $dbNamme vgroups 4
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT
tests/script/tsim/tmq/clearConsume.sim 0 → 100644
浏览文件 @ 312b7a4f
sql connect
#---- global parameters start ----#
$dbName = db
$vgroups = 1
$stbPrefix = stb
$ctbPrefix = ctb
$ntbPrefix = ntb
$stbNum = 1
$ctbNum = 10
$ntbNum = 10
$rowsPerCtb = 100
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----#
sql use $dbName
print == create consume info table and consume result table
sql drop table consumeinfo
sql drop table consumeresult
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables
if $rows != 2 then
return -1
endi
tests/script/tsim/tmq/consume.sh
浏览文件 @ 312b7a4f
......@@ -11,16 +11,25 @@ set +e
# set default value for parameters
EXEC_OPTON=start
DB_NAME=db
CDB_NAME=db
POLL_DELAY=5
VALGRIND=0
SIGNAL=SIGINT
SHOW_MSG=0
SHOW_ROW=0
while getopts "d:s:v:y:x:" arg
while getopts "d:s:v:y:x:g:r:w:" arg
do
case $arg in
d)
DB_NAME=$OPTARG
;;
g)
SHOW_MSG=$OPTARG
;;
r)
SHOW_ROW=$OPTARG
;;
s)
EXEC_OPTON=$OPTARG
;;
......@@ -33,6 +42,9 @@ do
x)
SIGNAL=$OPTARG
;;
w)
CDB_NAME=$OPTARG
;;
?)
echo "unkown argument"
;;
......@@ -80,11 +92,11 @@ echo "DB_NAME: $DB_NAME
echo "------------------------------------------------------------------------"
if [ "$EXEC_OPTON" = "start" ]; then
if [ $VALGRIND -eq 1 ]; then
echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 &
nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 &
echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 &
nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 &
else
echo "nohup $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 &"
nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME > /dev/null 2>&1 &
echo "nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &"
nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &
fi
else
PID=`ps -ef|grep tmq_sim | grep -v grep | awk '{print $2}'`
......
tests/script/tsim/tmq/prepareBasicEnv.sim 0 → 100644
浏览文件 @ 312b7a4f
# stop all dnodes before start this case
system sh/stop_dnodes.sh
# deploy dnode 1
system sh/deploy.sh -n dnode1 -i 1
# add some config items for this case
#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
# start dnode 1
system sh/exec.sh -n dnode1 -s start
sql connect
#---- global parameters start ----#
$dbName = db
$vgroups = 1
$stbPrefix = stb
$ctbPrefix = ctb
$ntbPrefix = ntb
$stbNum = 1
$ctbNum = 10
$ntbNum = 10
$rowsPerCtb = 100
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----#
print == create database $dbName vgroups $vgroups
sql create database $dbName vgroups $vgroups
#wait database ready
$loop_cnt = 0
check_db_ready:
if $loop_cnt == 10 then
print ====> database not ready!
return -1
endi
sql show databases
print ==> rows: $rows
print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12]
print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20]
if $data(db)[20] != nostrict then
sleep 100
$loop_cnt = $loop_cnt + 1
goto check_db_ready
endi
sql use $dbName
print == create consume info table and consume result table
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables
if $rows != 2 then
return -1
endi
print == create super table
sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int)
sql show stables
if $rows != 1 then
return -1
endi
print == create child table, normal table and insert data
$i = 0
while $i < $ctbNum
$ctb = $ctbPrefix . $i
$ntb = $ntbPrefix . $i
sql create table $ctb using $stbPrefix tags( $i )
sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16))
$x = 0
while $x < $rowsPerCtb
$binary = ' . binary-
$binary = $binary . $i
$binary = $binary . '
sql insert into $ctb values ($tstart , $i , $x , $binary )
sql insert into $ntb values ($tstart , $i , $x , $binary )
$tstart = $tstart + 1
$x = $x + 1
endw
$i = $i + 1
$tstart = 1640966400000
endw
tests/test/c/tmqSim.c
浏览文件 @ 312b7a4f
......@@ -43,6 +43,7 @@ typedef struct {
int64_t expectMsgCnt;
int64_t consumeMsgCnt;
int64_t consumeRowCnt;
int32_t checkresult;
char topicString[1024];
......@@ -62,8 +63,10 @@ typedef struct {
typedef struct {
// input from argvs
char cdbName[32];
char dbName[32];
int32_t showMsgFlag;
int32_t showRowFlag;
int32_t consumeDelay; // unit s
int32_t numOfThread;
SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT];
......@@ -85,51 +88,62 @@ static void printHelp() {
printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default ");
printf("%s%s\n", indent, "-g");
printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag);
printf("%s%s\n", indent, "-r");
printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag);
printf("%s%s\n", indent, "-y");
printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay);
exit(EXIT_SUCCESS);
}
void initLogFile() {
// FILE *fp = fopen(g_stConfInfo.resultFileName, "a");
TdFilePtr pFile = taosOpenFile("./tmqlog.txt", TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM);
char file[256];
sprintf(file, "%s/../log/tmqlog.txt", configDir);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM);
if (NULL == pFile) {
fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt");
exit - 1;
exit -1;
};
g_fp = pFile;
}
void saveConfigToLogFile() {
time_t tTime = taosGetTimestampSec();
struct tm tm = *taosLocalTime(&tTime, NULL);
taosFprintfFile(pFile, "###################################################################\n");
taosFprintfFile(pFile, "# configDir: %s\n", configDir);
taosFprintfFile(pFile, "# dbName: %s\n", g_stConfInfo.dbName);
taosFprintfFile(pFile, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag);
taosFprintfFile(pFile, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay);
taosFprintfFile(g_fp, "###################################################################\n");
taosFprintfFile(g_fp, "# configDir: %s\n", configDir);
taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName);
taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName);
taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag);
taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag);
taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay);
for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
taosFprintfFile(pFile, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
taosFprintfFile(pFile, " Topics: ");
for (int i = 0; i < g_stConfInfo.stThreads[i].numOfTopic; i++) {
taosFprintfFile(pFile, "%s, ", g_stConfInfo.stThreads[i].topics[i]);
taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
taosFprintfFile(g_fp, " Topics: ");
for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfTopic; i++) {
taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[i]);
}
taosFprintfFile(pFile, "\n");
taosFprintfFile(pFile, " Key: ");
for (int i = 0; i < g_stConfInfo.stThreads[i].numOfKey; i++) {
taosFprintfFile(pFile, "%s:%s, ", g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]);
taosFprintfFile(g_fp, "\n");
taosFprintfFile(g_fp, " Key: ");
for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfKey; i++) {
taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]);
}
taosFprintfFile(pFile, "\n");
taosFprintfFile(g_fp, "\n");
}
taosFprintfFile(pFile, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
taosFprintfFile(g_fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
taosFprintfFile(pFile, "###################################################################\n");
taosFprintfFile(g_fp, "###################################################################\n");
}
void parseArgument(int32_t argc, char* argv[]) {
memset(&g_stConfInfo, 0, sizeof(SConfInfo));
g_stConfInfo.showMsgFlag = 0;
g_stConfInfo.showRowFlag = 0;
g_stConfInfo.consumeDelay = 5;
for (int32_t i = 1; i < argc; i++) {
......@@ -138,10 +152,14 @@ void parseArgument(int32_t argc, char* argv[]) {
exit(0);
} else if (strcmp(argv[i], "-d") == 0) {
strcpy(g_stConfInfo.dbName, argv[++i]);
} else if (strcmp(argv[i], "-w") == 0) {
strcpy(g_stConfInfo.cdbName, argv[++i]);
} else if (strcmp(argv[i], "-c") == 0) {
strcpy(configDir, argv[++i]);
} else if (strcmp(argv[i], "-g") == 0) {
g_stConfInfo.showMsgFlag = atol(argv[++i]);
} else if (strcmp(argv[i], "-r") == 0) {
g_stConfInfo.showRowFlag = atol(argv[++i]);
} else if (strcmp(argv[i], "-y") == 0) {
g_stConfInfo.consumeDelay = atol(argv[++i]);
} else {
......@@ -150,11 +168,17 @@ void parseArgument(int32_t argc, char* argv[]) {
}
}
initLogFile();
taosFprintfFile(g_fp, "====parseArgument() success\n");
#if 1
pPrint("%s configDir:%s %s", GREEN, configDir, NC);
pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC);
pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC);
pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC);
pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC);
pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC);
#endif
}
......@@ -181,23 +205,28 @@ void ltrim(char* str) {
}
static int running = 1;
static void msg_process(TAOS_RES* msg, int32_t msgIndex, int32_t threadLable) {
static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) {
char buf[1024];
int32_t totalRows = 0;
// printf("topic: %s\n", tmq_get_topic_name(msg));
// printf("vg:%d\n", tmq_get_vgroup_id(msg));
taosFprintfFile(g_fp, "msg index:%d, threadLable: %d\n", msgIndex, threadLable);
//printf("topic: %s\n", tmq_get_topic_name(msg));
//printf("vg:%d\n", tmq_get_vgroup_id(msg));
taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable);
taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg));
while (1) {
TAOS_ROW row = taos_fetch_row(msg);
if (row == NULL) break;
if (0 != g_stConfInfo.showRowFlag) {
TAOS_FIELD* fields = taos_fetch_fields(msg);
int32_t numOfFields = taos_field_count(msg);
// taos_print_row(buf, row, fields, numOfFields);
// printf("%s\n", buf);
// taosFprintfFile(g_fp, "%s\n", buf);
taos_print_row(buf, row, fields, numOfFields);
taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf);
}
totalRows++;
}
return totalRows;
}
int queryDB(TAOS* taos, char* command) {
......@@ -213,31 +242,43 @@ int queryDB(TAOS* taos, char* command) {
return 0;
}
void build_consumer(SThreadInfo* pInfo) {
char sqlStr[1024] = {0};
static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) {
printf("tmq_commit_cb_print() commit %d\n", resp);
}
TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
assert(pConn != NULL);
void build_consumer(SThreadInfo *pInfo) {
tmq_conf_t* conf = tmq_conf_new();
sprintf(sqlStr, "use %s", g_stConfInfo.dbName);
TAOS_RES* pRes = taos_query(pConn, sqlStr);
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
exit(-1);
}
taos_free_result(pRes);
//tmq_conf_set(conf, "td.connect.ip", "localhost");
//tmq_conf_set(conf, "td.connect.port", "6030");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_t* conf = tmq_conf_new();
// tmq_conf_set(conf, "group.id", "tg2");
tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
// tmq_conf_set(conf, "group.id", "cgrp1");
for (int32_t i = 0; i < pInfo->numOfKey; i++) {
tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]);
}
//tmq_conf_set(conf, "client.id", "c-001");
//tmq_conf_set(conf, "enable.auto.commit", "true");
//tmq_conf_set(conf, "enable.auto.commit", "false");
//tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
//tmq_conf_set(conf, "auto.offset.reset", "none");
//tmq_conf_set(conf, "auto.offset.reset", "earliest");
//tmq_conf_set(conf, "auto.offset.reset", "latest");
pInfo->tmq = tmq_consumer_new(conf, NULL, 0);
return;
}
void build_topic_list(SThreadInfo* pInfo) {
void build_topic_list(SThreadInfo *pInfo) {
pInfo->topicList = tmq_list_new();
// tmq_list_append(topic_list, "test_stb_topic_1");
for (int32_t i = 0; i < pInfo->numOfTopic; i++) {
......@@ -246,15 +287,19 @@ void build_topic_list(SThreadInfo* pInfo) {
return;
}
int32_t saveConsumeResult(SThreadInfo* pInfo) {
int32_t saveConsumeResult(SThreadInfo *pInfo) {
char sqlStr[1024] = {0};
TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
assert(pConn != NULL);
// schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %d)", g_stConfInfo.dbName,
pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->checkresult);
sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)",
g_stConfInfo.cdbName,
pInfo->consumerId,
pInfo->consumeMsgCnt,
pInfo->consumeRowCnt,
pInfo->checkresult);
TAOS_RES* pRes = taos_query(pConn, sqlStr);
if (taos_errno(pRes) != 0) {
......@@ -268,17 +313,17 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) {
return 0;
}
void loop_consume(SThreadInfo* pInfo) {
void loop_consume(SThreadInfo *pInfo) {
tmq_resp_err_t err;
int64_t totalMsgs = 0;
// int64_t totalRows = 0;
int64_t totalRows = 0;
while (running) {
TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000);
if (tmqMsg) {
if (0 != g_stConfInfo.showMsgFlag) {
msg_process(tmqMsg, totalMsgs, 0);
totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId);
}
taos_free_result(tmqMsg);
......@@ -300,16 +345,20 @@ void loop_consume(SThreadInfo* pInfo) {
}
pInfo->consumeMsgCnt = totalMsgs;
pInfo->consumeRowCnt = totalRows;
taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %"PRId64", consumeRowCnt: %"PRId64"\n", pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt);
}
void* consumeThreadFunc(void* param) {
void *consumeThreadFunc(void *param) {
int32_t totalMsgs = 0;
SThreadInfo* pInfo = (SThreadInfo*)param;
SThreadInfo *pInfo = (SThreadInfo *)param;
build_consumer(pInfo);
build_topic_list(pInfo);
if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) {
if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)){
return NULL;
}
......@@ -376,10 +425,12 @@ int32_t getConsumeInfo() {
TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
assert(pConn != NULL);
sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.dbName);
sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName);
TAOS_RES* pRes = taos_query(pConn, sqlStr);
if (taos_errno(pRes) != 0) {
printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes));
taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes));
taosCloseFile(&g_fp);
taos_free_result(pRes);
exit(-1);
}
......@@ -388,8 +439,7 @@ int32_t getConsumeInfo() {
int num_fields = taos_num_fields(pRes);
TAOS_FIELD* fields = taos_fetch_fields(pRes);
// schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint,
// ifcheckdata int
// schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int
int32_t numOfThread = 0;
while ((row = taos_fetch_row(pRes))) {
......@@ -401,18 +451,18 @@ int32_t getConsumeInfo() {
}
if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]);
g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t *)row[i]);
} else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]);
} else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]);
} else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) {
g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]);
g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t *)row[i]);
} else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]);
g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t *)row[i]);
}
}
numOfThread++;
numOfThread ++;
}
g_stConfInfo.numOfThread = numOfThread;
......@@ -423,10 +473,11 @@ int32_t getConsumeInfo() {
return 0;
}
int main(int32_t argc, char* argv[]) {
parseArgument(argc, argv);
getConsumeInfo();
initLogFile();
saveConfigToLogFile();
TdThreadAttr thattr;
taosThreadAttrInit(&thattr);
......@@ -434,15 +485,14 @@ int main(int32_t argc, char* argv[]) {
// pthread_create one thread to consume
for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) {
taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc,
(void*)(&(g_stConfInfo.stThreads[i])));
taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, (void *)(&(g_stConfInfo.stThreads[i])));
}
for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL);
}
// printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt);
//printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt);
taosFprintfFile(g_fp, "\n");
taosCloseFile(&g_fp);
......
tools/shell/src/shellEngine.c
浏览文件 @ 312b7a4f
......@@ -224,27 +224,63 @@ int32_t shellRunCommand(TAOS *con, char *command) {
}
}
char quote = 0, *cmd = command;
bool esc = false;
char quote = 0, *cmd = command, *p = command;
for (char c = *command++; c != 0; c = *command++) {
if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) {
command ++;
if (esc) {
switch (c) {
case 'n':
c = '\n';
break;
case 'r':
c = '\r';
break;
case 't':
c = '\t';
break;
case 'G':
*p++ = '\\';
break;
case '\'':
case '"':
if (quote) {
*p++ = '\\';
}
break;
}
*p++ = c;
esc = false;
continue;
}
if (c == '\\') {
if (quote != 0 && (*command == '_' || *command == '\\')) {
// DO nothing
} else {
esc = true;
continue;
}
}
if (quote == c) {
quote = 0;
} else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) {
} else if (quote == 0 && (c == '\'' || c == '"')) {
quote = c;
} else if (c == ';' && quote == 0) {
c = *command;
*command = 0;
}
*p++ = c;
if (c == ';' && quote == 0) {
c = *p;
*p = 0;
if (shellRunSingleCommand(con, cmd) < 0) {
return -1;
}
*command = c;
cmd = command;
*p = c;
p = cmd;
}
}
*p = 0;
return shellRunSingleCommand(con, cmd);
}
......@@ -538,23 +574,19 @@ static void shellPrintNChar(const char *str, int length, int width) {
while (pos < length) {
TdWchar wc;
int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
if (bytes <= 0) {
if (bytes == 0) {
break;
}
if (pos + bytes > length) {
pos += bytes;
if (pos > length) {
break;
}
int w = 0;
#ifdef WINDOWS
w = bytes;
int w = bytes;
#else
if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
w = bytes;
}else{
w = taosWcharWidth(wc);
}
int w = taosWcharWidth(wc);
#endif
pos += bytes;
if (w <= 0) {
continue;
}
......@@ -648,7 +680,6 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
shellPrintNChar(val, length, width);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
......@@ -761,8 +792,7 @@ static int calcColWidth(TAOS_FIELD *field, int precision) {
return TMAX(field->bytes, width);
}
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:{
case TSDB_DATA_TYPE_NCHAR: {
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > tsMaxBinaryDisplayWidth) {
return TMAX(tsMaxBinaryDisplayWidth, width);
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或