diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 074014b0d7ba89cd7a712bbd88e13fe2952e20fa..21b8b661df9809d8631cf1e7e915eb78c500671e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -193,6 +193,7 @@ endif(${BUILD_WITH_TRAFT}) # LIBUV if(${BUILD_WITH_UV}) + add_compile_options(-Wno-sign-compare) add_subdirectory(libuv) endif(${BUILD_WITH_UV}) diff --git a/example/src/tmq.c b/example/src/tmq.c index 26e5ea82c19a3f0ac90689a4b4f1362caa193ec6..094fd94bfc00a137073dae63a41e4754fb922ced 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -66,7 +66,7 @@ int32_t init_env() { } int32_t create_topic() { - printf("create topic"); + printf("create topic\n"); TAOS_RES* pRes; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -91,6 +91,10 @@ int32_t create_topic() { return 0; } +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { + printf("commit %d\n", resp); +} + tmq_t* build_consumer() { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -103,6 +107,7 @@ tmq_t* build_consumer() { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); return tmq; } @@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - static const int MIN_COMMIT_COUNT = 1000; + static const int MIN_COMMIT_COUNT = 1; int msg_count = 0; tmq_resp_err_t err; @@ -214,6 +219,6 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); /*perf_loop(tmq, topic_list);*/ - basic_consume_loop(tmq, topic_list); - /*sync_consume_loop(tmq, topic_list);*/ + /*basic_consume_loop(tmq, topic_list);*/ + sync_consume_loop(tmq, topic_list); } diff --git a/include/client/taos.h b/include/client/taos.h index a960d379375acca491c7ec1b4b0541e3dae3489e..2c8135c8ffaf5e920e76544f5a0796df6d5d0eda 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi /* --------------------------TMQ INTERFACE------------------------------- */ enum tmq_resp_err_t { + TMQ_RESP_ERR__FAIL = -1, TMQ_RESP_ERR__SUCCESS = 0, - TMQ_RESP_ERR__FAIL = 1, }; typedef enum tmq_resp_err_t tmq_resp_err_t; @@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); #if 0 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); -DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics); #endif DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); @@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t * #if 0 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); #endif +DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ enum tmq_conf_res_t { diff --git a/include/common/common.h b/include/common/common.h index fd5b6717abb9d7bd1cf3b35e8eda56c3194a523f..8fa2d03d6d3f79a220b6c10a38070a8a42026155 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -16,7 +16,6 @@ #ifndef TDENGINE_COMMON_H #define TDENGINE_COMMON_H - #ifdef __cplusplus extern "C" { #endif @@ -43,14 +42,16 @@ extern "C" { // int16_t bytes; // } SSchema; -#define TMQ_REQ_TYPE_COMMIT_ONLY 0 -#define TMQ_REQ_TYPE_CONSUME_ONLY 1 -#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 +enum { + TMQ_CONF__RESET_OFFSET__LATEST = -1, + TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, + TMQ_CONF__RESET_OFFSET__NONE = -3, +}; typedef struct { uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid + SArray* pGroupList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; typedef struct SColumnDataAgg { @@ -79,14 +80,14 @@ typedef struct SConstantItem { // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg *pBlockAgg; - SArray *pDataBlock; // SArray - SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. - SDataBlockInfo info; + SColumnDataAgg* pBlockAgg; + SArray* pDataBlock; // SArray + SArray* pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { - int32_t *offset; // start position for each entry in the list + int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data uint32_t allocLen; // allocated buffer size } SVarColAttr; @@ -94,11 +95,11 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed - bool hasNull;// if current column data has null value. - char *pData; // the corresponding block data in memory + SColumnInfo info; // TODO filter info needs to be removed + bool hasNull; // if current column data has null value. + char* pData; // the corresponding block data in memory union { - char *nullbitmap; // bitmap, one bit for each item in the list + char* nullbitmap; // bitmap, one bit for each item in the list SVarColAttr varmeta; }; } SColumnInfoData; @@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeFixedI64(buf, pRsp->committedOffset); tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); @@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeFixedI64(buf, &pRsp->committedOffset); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); @@ -250,11 +249,11 @@ typedef struct SSqlExpr { char token[TSDB_COL_NAME_LEN]; // original token SSchema resSchema; - int32_t numOfCols; - SColumn* pColumns; // data columns that are required by query - int32_t interBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - SVariant param[3]; // parameters are not more than 3 + int32_t numOfCols; + SColumn* pColumns; // data columns that are required by query + int32_t interBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + SVariant param[3]; // parameters are not more than 3 } SSqlExpr; typedef struct SExprInfo { @@ -271,7 +270,7 @@ typedef struct SSessionWindow { SColumn col; } SSessionWindow; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) diff --git a/include/common/tep.h b/include/common/tep.h index 584b8a5a71fb7833ea78c9991933c042099b237f..6d09d0ce51e3af5f51461c27003025a8b58546f1 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -23,7 +23,7 @@ typedef struct SBlockOrderInfo { // bool hasNull; } SBlockOrderInfo; -int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); +int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp *pEp); void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); @@ -68,8 +68,8 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } #define colDataGet(p1_, r_) \ - ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \ - : (p1_)->pData + ((r_) * (p1_)->info.bytes)); + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ + : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 4bc8c9fb86c03182af16b6ea0f38248982243a04..bb704f153644b5d469e429de753cf550dcd709d3 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -21,35 +21,11 @@ extern "C" { #endif #include "tdef.h" -#include "tcfg.h" - -// cluster -extern char tsFirst[]; -extern char tsSecond[]; -extern char tsLocalFqdn[]; -extern char tsLocalEp[]; -extern uint16_t tsServerPort; -extern int32_t tsStatusInterval; -extern int8_t tsEnableTelemetryReporting; -extern int32_t tsNumOfSupportVnodes; // common -extern int tsRpcTimer; -extern int tsRpcMaxTime; -extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled -extern int32_t tsMaxConnections; -extern int32_t tsMaxShellConns; -extern int32_t tsShellActivityTimer; -extern uint32_t tsMaxTmrCtrl; -extern float tsNumOfThreadsPerCore; -extern int32_t tsNumOfCommitThreads; -extern float tsRatioOfQueryCores; -extern int8_t tsDaylight; -extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; -extern char tsTempDir[]; extern int tsCompatibleModel; // 2.0 compatible model extern int8_t tsEnableSlaveQuery; extern int8_t tsEnableAdjustMaster; @@ -66,7 +42,6 @@ extern int8_t tsDeadLockKillQuery; // client extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxRegexStringLen; -extern int8_t tsTscEnableRecordSql; extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; @@ -77,41 +52,8 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -// system info -extern float tsTotalLogDirGB; -extern float tsTotalTmpDirGB; -extern float tsTotalDataDirGB; -extern float tsAvailLogDirGB; -extern float tsAvailTmpDirectorySpace; -extern float tsAvailDataDirGB; -extern float tsUsedDataDirGB; -extern float tsMinimalLogDirGB; -extern float tsReservedTmpDirectorySpace; -extern float tsMinimalDataDirGB; -extern uint32_t tsVersion; - -// build info -extern char version[]; -extern char compatible_version[]; -extern char gitinfo[]; -extern char gitinfoOfInternal[]; -extern char buildinfo[]; - -// lossy -extern char tsLossyColumns[]; -extern double tsFPrecision; -extern double tsDPrecision; -extern uint32_t tsMaxRange; -extern uint32_t tsCurRange; -extern char tsCompressor[]; - -extern int32_t tsDiskCfgNum; -extern SDiskCfg tsDiskCfg[]; - #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) -void taosInitGlobalCfg(); -int32_t taosCheckAndPrintCfg(); int32_t taosCfgDynamicOptions(char *msg); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5164fb6ccd2323764ac3aec259559ee572fbde4f..9ab78f3c9603011296c7ef450ba0d792dbc19043 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -422,8 +422,8 @@ typedef struct { } SColumnInfo; typedef struct { - uint64_t uid; - TSKEY key; // last accessed ts, for subscription + int64_t uid; + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -554,8 +554,8 @@ int32_t tSerializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; } SDropDbRsp; int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); @@ -570,12 +570,12 @@ int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; - int32_t vgVersion; - int32_t vgNum; - int8_t hashMethod; - SArray* pVgroupInfos; // Array of SVgroupInfo + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; + SArray* pVgroupInfos; // Array of SVgroupInfo } SUseDbRsp; int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); @@ -656,9 +656,9 @@ int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp typedef struct { int32_t statusInterval; int64_t checkTime; // 1970-01-01 00:00:00.000 - char timezone[TSDB_TIMEZONE_LEN]; // tsTimezone - char locale[TSDB_LOCALE_LEN]; // tsLocale - char charset[TSDB_LOCALE_LEN]; // tsCharset + char timezone[TD_TIMEZONE_LEN]; // tsTimezone + char locale[TD_LOCALE_LEN]; // tsLocale + char charset[TD_LOCALE_LEN]; // tsCharset } SClusterCfg; typedef struct { @@ -725,7 +725,7 @@ typedef struct { int32_t vgId; int32_t dnodeId; char db[TSDB_DB_FNAME_LEN]; - uint64_t dbUid; + int64_t dbUid; int32_t vgVersion; int32_t cacheBlockSize; int32_t totalBlocks; @@ -753,10 +753,10 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); typedef struct { - int32_t vgId; - int32_t dnodeId; - uint64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; + int32_t vgId; + int32_t dnodeId; + int64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; } SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); @@ -796,7 +796,7 @@ typedef struct { char tbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; - uint64_t dbId; + int64_t dbId; int32_t numOfTags; int32_t numOfColumns; int8_t precision; @@ -1268,7 +1268,7 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; int32_t sverson; int32_t execLen; char* executor; @@ -1279,11 +1279,11 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; } SDDropTopicReq; typedef struct SVCreateTbReq { - uint64_t ver; // use a general definition + int64_t ver; // use a general definition char* name; uint32_t ttl; uint32_t keep; @@ -1314,8 +1314,8 @@ int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); typedef struct { - uint64_t ver; // use a general definition - SArray* pArray; + int64_t ver; // use a general definition + SArray* pArray; } SVCreateTbBatchReq; typedef struct { @@ -1325,7 +1325,7 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); typedef struct { - uint64_t ver; + int64_t ver; char* name; uint8_t type; tb_uid_t suid; @@ -1760,35 +1760,19 @@ typedef struct { char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqOffset; -typedef struct { - int32_t vgId; - SArray* offsets; // SArray -} SMqVgOffsets; - typedef struct { int32_t num; SMqOffset* offsets; -} SMqCMResetOffsetReq; - -typedef struct { - int32_t reserved; -} SMqCMResetOffsetRsp; - -typedef struct { - int64_t leftForVer; - SMqVgOffsets offsets; -} SMqMVResetOffsetReq; +} SMqCMCommitOffsetReq; typedef struct { int32_t reserved; -} SMqMVResetOffsetRsp; +} SMqCMCommitOffsetRsp; int32_t tEncodeSMqOffset(SCoder* encoder, const SMqOffset* pOffset); int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset); -int32_t tEncodeSMqCMResetOffsetReq(SCoder* encoder, const SMqCMResetOffsetReq* pReq); -int32_t tDecodeSMqCMResetOffsetReq(SCoder* decoder, SMqCMResetOffsetReq* pReq); -int32_t tEncodeSMqMVResetOffsetReq(SCoder* encoder, const SMqMVResetOffsetReq* pReq); -int32_t tDecodeSMqMVResetOffsetReq(SCoder* decoder, SMqMVResetOffsetReq* pReq); +int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq); +int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq); typedef struct { uint32_t nCols; @@ -1870,7 +1854,6 @@ typedef struct { typedef struct { int64_t consumerId; SSchemaWrapper* schemas; - int64_t committedOffset; int64_t reqOffset; int64_t rspOffset; int32_t skipLogNum; @@ -1881,22 +1864,18 @@ typedef struct { // one req for one vg+topic typedef struct { SMsgHead head; - // 0: commit only, current offset - // 1: consume only, poll next offset - // 2: commit current and consume next offset - int32_t reqType; - int64_t reqId; int64_t consumerId; int64_t blockingTime; char cgroup[TSDB_CONSUMER_GROUP_LEN]; - int64_t offset; + int64_t currentOffset; char topic[TSDB_TOPIC_FNAME_LEN]; } SMqConsumeReq; typedef struct { int32_t vgId; + int64_t offset; SEpSet epSet; } SMqSubVgEp; @@ -1917,12 +1896,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taos static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return buf; } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1a63ea73a597d215abb3a8f0b43922da67bcb647..a3e44968f76fd43a161d7cd33f4ea5fc0304041b 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -135,18 +135,19 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) @@ -194,4 +195,3 @@ enum { TDMT_MAX #endif }; -// clang-format on diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 5a1e442277b7e5ceacfdb4c098cd5c09dddea8f1..39a5f0e2b8a42f96390b060929c9c4bcd10beeb6 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -91,6 +91,8 @@ typedef struct { do { \ switch (_type) { \ case TSDB_DATA_TYPE_BOOL: \ + *(bool *)(_v) = (bool)(_data); \ + break; \ case TSDB_DATA_TYPE_TINYINT: \ *(int8_t *)(_v) = (int8_t)(_data); \ break; \ @@ -104,6 +106,7 @@ typedef struct { *(uint16_t *)(_v) = (uint16_t)(_data); \ break; \ case TSDB_DATA_TYPE_BIGINT: \ + case TSDB_DATA_TYPE_TIMESTAMP: \ *(int64_t *)(_v) = (int64_t)(_data); \ break; \ case TSDB_DATA_TYPE_UBIGINT: \ diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index e64d358de442be20ebf07d85b0ec4597c07ef14b..d630fd0b1acf8beedce951d533a18440b49bed11 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -27,15 +27,18 @@ typedef struct SDnode SDnode; /* ------------------------ Environment ------------------ */ typedef struct { - int32_t sver; - int32_t numOfCores; - int16_t numOfCommitThreads; - int8_t enableTelem; - char timezone[TSDB_TIMEZONE_LEN]; - char locale[TSDB_LOCALE_LEN]; - char charset[TSDB_LOCALE_LEN]; - char buildinfo[64]; - char gitinfo[48]; + int32_t sver; + int32_t numOfCores; + uint16_t numOfCommitThreads; + bool enableTelem; + bool printAuth; + int32_t rpcTimer; + int32_t rpcMaxTime; + char timezone[TD_TIMEZONE_LEN]; + char locale[TD_LOCALE_LEN]; + char charset[TD_LOCALE_LEN]; + char buildinfo[64]; + char gitinfo[48]; } SDnodeEnvCfg; /** @@ -65,6 +68,7 @@ typedef struct { char localEp[TSDB_EP_LEN]; char localFqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; + char secondEp[TSDB_EP_LEN]; } SDnodeObjCfg; /** diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index e1306800d3e3c9491bad297130d23ba83932ec6e..01855f6d92c2a5aa37cbb2a82de6d4236037cfbd 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -46,7 +46,8 @@ typedef struct SMnodeLoad { typedef struct SMnodeCfg { int32_t sver; - int8_t enableTelem; + bool enableTelem; + bool printAuth; int32_t statusInterval; int32_t shellActivityTimer; char *timezone; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 1e967a6d2bc7e08a846c6619f94352f6e05c2e5e..bf48a8523c427a10801b0cccaf1046d08c9cc07f 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -113,14 +113,15 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_SUBSCRIBE = 10, - SDB_CONSUMER = 11, - SDB_TOPIC = 12, - SDB_VGROUP = 13, - SDB_STB = 14, - SDB_DB = 15, - SDB_FUNC = 16, - SDB_MAX = 17 + SDB_OFFSET = 10, + SDB_SUBSCRIBE = 11, + SDB_CONSUMER = 12, + SDB_TOPIC = 13, + SDB_VGROUP = 14, + SDB_STB = 15, + SDB_DB = 16, + SDB_FUNC = 17, + SDB_MAX = 18 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/libs/config/config.h b/include/libs/config/config.h new file mode 100644 index 0000000000000000000000000000000000000000..594c2586484cbbce46836bc4e7aae6a198a24fd0 --- /dev/null +++ b/include/libs/config/config.h @@ -0,0 +1,105 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_CONFIG_H_ +#define _TD_CONFIG_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define CFG_NAME_MAX_LEN 128 + +typedef enum { + CFG_STYPE_DEFAULT, + CFG_STYPE_CFG_FILE, + CFG_STYPE_ENV_FILE, + CFG_STYPE_ENV_VAR, + CFG_STYPE_APOLLO_URL, + CFG_STYPE_ARG_LIST, + CFG_STYPE_API_OPTION +} ECfgSrcType; + +typedef enum { + CFG_DTYPE_NONE, + CFG_DTYPE_BOOL, + CFG_DTYPE_INT32, + CFG_DTYPE_INT64, + CFG_DTYPE_FLOAT, + CFG_DTYPE_STRING, + CFG_DTYPE_IPSTR, + CFG_DTYPE_DIR, + CFG_DTYPE_LOCALE, + CFG_DTYPE_CHARSET, + CFG_DTYPE_TIMEZONE +} ECfgDataType; + +typedef struct SConfigItem { + ECfgSrcType stype; + ECfgDataType dtype; + char *name; + union { + bool bval; + float fval; + int32_t i32; + int64_t i64; + char *str; + }; + union { + int64_t imin; + double fmin; + }; + union { + int64_t imax; + double fmax; + }; +} SConfigItem; + +typedef struct SConfig SConfig; + +SConfig *cfgInit(); +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); +void cfgCleanup(SConfig *pCfg); + +int32_t cfgGetSize(SConfig *pCfg); +SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter); +void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter); +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype); + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVa); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); + +const char *cfgStypeStr(ECfgSrcType type); +const char *cfgDtypeStr(ECfgDataType type); + +void cfgDumpCfg(SConfig *pCfg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_CONFIG_H_*/ diff --git a/include/libs/function/function.h b/include/libs/function/function.h index bb5d51686d44bcf14da376c9dbd545b98e471cc9..e970a0d693db322a4cfa8a38e5aa28f778ab4197 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -228,6 +228,7 @@ typedef struct SAggFunctionInfo { typedef struct SScalarParam { void* data; + bool colData; int32_t num; int32_t type; int32_t bytes; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 59cd672765468781454ba4a5f64f55e152885ab9..c54e1b3ad039af17aa85316ab3236e75439b4707 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -90,45 +90,6 @@ typedef struct SValueNode { } datum; } SValueNode; -typedef enum EOperatorType { - // arithmetic operator - OP_TYPE_ADD = 1, - OP_TYPE_SUB, - OP_TYPE_MULTI, - OP_TYPE_DIV, - OP_TYPE_MOD, - - // bit operator - OP_TYPE_BIT_AND, - OP_TYPE_BIT_OR, - - // comparison operator - OP_TYPE_GREATER_THAN, - OP_TYPE_GREATER_EQUAL, - OP_TYPE_LOWER_THAN, - OP_TYPE_LOWER_EQUAL, - OP_TYPE_EQUAL, - OP_TYPE_NOT_EQUAL, - OP_TYPE_IN, - OP_TYPE_NOT_IN, - OP_TYPE_LIKE, - OP_TYPE_NOT_LIKE, - OP_TYPE_MATCH, - OP_TYPE_NMATCH, - OP_TYPE_IS_NULL, - OP_TYPE_IS_NOT_NULL, - OP_TYPE_IS_TRUE, - OP_TYPE_IS_FALSE, - OP_TYPE_IS_UNKNOWN, - OP_TYPE_IS_NOT_TRUE, - OP_TYPE_IS_NOT_FALSE, - OP_TYPE_IS_NOT_UNKNOWN, - - // json operator - OP_TYPE_JSON_GET_VALUE, - OP_TYPE_JSON_CONTAINS -} EOperatorType; - typedef struct SOperatorNode { SExprNode node; // QUERY_NODE_OPERATOR EOperatorType opType; @@ -136,11 +97,6 @@ typedef struct SOperatorNode { SNode* pRight; } SOperatorNode; -typedef enum ELogicConditionType { - LOGIC_COND_TYPE_AND, - LOGIC_COND_TYPE_OR, - LOGIC_COND_TYPE_NOT, -} ELogicConditionType; typedef struct SLogicConditionNode { SExprNode node; // QUERY_NODE_LOGIC_CONDITION diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 8db74f4587a7160ac12a2bf70a2b69e4a9d508e5..fedb487931a6e721a49ba0194716f457aa1b433e 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -20,6 +20,14 @@ extern "C" { #endif typedef struct SFilterInfo SFilterInfo; +typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); + + +enum { + FLT_OPTION_NO_REWRITE = 1, + FLT_OPTION_TIMESTAMP = 2, + FLT_OPTION_NEED_UNIQE = 4, +}; typedef struct SFilterColumnParam{ int32_t numOfCols; @@ -27,8 +35,18 @@ typedef struct SFilterColumnParam{ } SFilterColumnParam; +extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); +extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); +extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); +extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param); +extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); +extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); +extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); +extern void filterFreeInfo(SFilterInfo *info); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); + #ifdef __cplusplus } #endif -#endif // TDENGINE_FILTER_H \ No newline at end of file +#endif // TDENGINE_FILTER_H diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 4dbdb2e8bb8937849f24e15b9d7e08c67ea2e149..1902c11cc68097b63141d640ce7a8bdf4656d713 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -29,6 +29,7 @@ typedef struct SFilterInfo SFilterInfo; int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); int32_t scalarGetOperatorParamNum(EOperatorType type); +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 538aeb1a0e7927469cf90721dbbc16140c181e93..befd309582bb6452c5475c97387a5a89fffdf7e3 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -78,12 +78,21 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + // call back to keep conn or not + bool (*pfp)(void *parent, tmsg_t msgType); + void *parent; } SRpcInit; -int32_t rpcInit(); +typedef struct { + int32_t rpcTimer; + int32_t rpcMaxTime; + int32_t sver; +} SRpcCfg; + +int32_t rpcInit(SRpcCfg *pCfg); void rpcCleanup(); -void * rpcOpen(const SRpcInit *pRpc); +void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); void * rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); diff --git a/include/os/os.h b/include/os/os.h index 023d2b04705074abcec4c1dc927211bf63512112..d96f7efb51dd0ca36d307308ba005d4fd7f628b6 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -59,6 +59,7 @@ extern "C" { #include "osEndian.h" #include "osEnv.h" #include "osFile.h" +#include "osLocale.h" #include "osLz4.h" #include "osMath.h" #include "osMemory.h" @@ -73,6 +74,7 @@ extern "C" { #include "osThread.h" #include "osTime.h" #include "osTimer.h" +#include "osTimezone.h" void osInit(); diff --git a/include/os/osDir.h b/include/os/osDir.h index 35c294af39bf49967ab6fa1f15bc3bf35a523911..e328c420638eb1d146abe97f8abf6bacb6286377 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -23,8 +23,8 @@ extern "C" { void taosRemoveDir(const char *dirname); int32_t taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); -void taosRemoveOldFiles(char *dirname, int32_t keepDays); -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen); +void taosRemoveOldFiles(const char *dirname, int32_t keepDays); +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); #ifdef __cplusplus diff --git a/include/os/osEnv.h b/include/os/osEnv.h index a7fd86776c06ea6f399f6094ac52a78c0345a7e5..29898953e6667056e653562950d36514c4a60d42 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -16,16 +16,38 @@ #ifndef _TD_OS_ENV_H_ #define _TD_OS_ENV_H_ +#include "osSysinfo.h" + #ifdef __cplusplus extern "C" { #endif -extern char tsOsName[]; -extern char tsDataDir[]; -extern char tsLogDir[]; -extern char tsScriptDir[]; +typedef struct SOsEnv SOsEnv; + extern char configDir[]; +void osInit(); +void osUpdate(); + +bool osLogSpaceAvailable(); +int8_t osDaylight(); + +const char *osLogDir(); +const char *osTempDir(); +const char *osDataDir(); +const char *osName(); +const char *osTimezone(); +const char *osLocale(); +const char *osCharset(); + +void osSetLogDir(const char *logDir); +void osSetTempDir(const char *tempDir); +void osSetDataDir(const char *dataDir); +void osSetLogReservedSpace(float sizeInGB); +void osSetTempReservedSpace(float sizeInGB); +void osSetDataReservedSpace(float sizeInGB); +void osSetTimezone(const char *timezone); + #ifdef __cplusplus } #endif diff --git a/include/os/osLocale.h b/include/os/osLocale.h new file mode 100644 index 0000000000000000000000000000000000000000..6e313eb8cde14e67f56a9e8334239ca714b88afd --- /dev/null +++ b/include/os/osLocale.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_LOCALE_H_ +#define _TD_OS_LOCALE_H_ + +#include "os.h" +#include "osString.h" + +#ifdef __cplusplus +extern "C" { +#endif + +char *taosCharsetReplace(char *charsetstr); +void taosGetSystemLocale(char *outLocale, char *outCharset); +void taosSetSystemLocale(const char *inLocale, const char *inCharSet); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_LOCALE_H_*/ diff --git a/include/os/osSocket.h b/include/os/osSocket.h index d76b286d651b0124fd5db4fd4018f566f498a0d0..af99e4b47419350e408d8809e8d593d0b2a3996a 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -55,7 +55,7 @@ void taosSetMaskSIGPIPE(); int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); -uint32_t taosInetAddr(char *ipAddr); +uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) diff --git a/include/os/osString.h b/include/os/osString.h index 582411d4440e7b1539c4b926413c7ed9751dfe62..88160dd69e419450d1ec481a83a28065df99186d 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); bool taosValidateEncodec(const char *encodec); -char * taosCharsetReplace(char *charsetstr); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 9dcb075489ac3fed02c3bdb77610a420800c6859..64362603fea44648d159b1b2e3491c245066672f 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -21,18 +21,10 @@ extern "C" { #endif #include "os.h" - -#define TSDB_LOCALE_LEN 64 -#define TSDB_TIMEZONE_LEN 96 -extern int64_t tsPageSize; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern int32_t tsTotalMemoryMB; -extern char tsTimezone[]; -extern char tsLocale[]; -extern char tsCharset[]; // default encode string +#define TD_LOCALE_LEN 64 +#define TD_CHARSET_LEN 64 +#define TD_TIMEZONE_LEN 96 typedef struct { int64_t total; @@ -40,6 +32,19 @@ typedef struct { int64_t avail; } SDiskSize; +typedef struct SDiskSpace { + int64_t reserved; + SDiskSize size; +} SDiskSpace; + +extern int64_t tsPageSize; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern int32_t tsNumOfCores; +extern int32_t tsTotalMemoryMB; + + + int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetCpuCores(); void taosGetSystemInfo(); diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h new file mode 100644 index 0000000000000000000000000000000000000000..ff015ef0b1221eeba361818f5e010bd7f43fb0fd --- /dev/null +++ b/include/os/osTimezone.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_TIMEZONE_H_ +#define _TD_OS_TIMEZONE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void taosGetSystemTimezone(char *outTimezone); +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_TIMEZONE_H_*/ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 92028b08378af35926d7c9215ecc95fb5c63b5f3..2f7d4065697c394ed26bb21ebba11db163c96872 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -20,6 +20,8 @@ extern "C" { #endif +// clang-format off + #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) @@ -71,6 +73,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) #define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) +#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) @@ -260,6 +264,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) #define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) +#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) // dnode diff --git a/include/util/tconfig.h b/include/util/tconfig.h deleted file mode 100644 index 7dee01247b9e081585ab1ba4c4a1f907dec9cbc6..0000000000000000000000000000000000000000 --- a/include/util/tconfig.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_CONFIG_H -#define _TD_UTIL_CONFIG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TSDB_CFG_MAX_NUM 115 -#define TSDB_CFG_PRINT_LEN 23 -#define TSDB_CFG_OPTION_LEN 24 -#define TSDB_CFG_VALUE_LEN 41 - -#define TSDB_CFG_CTYPE_B_CONFIG 1U // can be configured from file -#define TSDB_CFG_CTYPE_B_SHOW 2U // can displayed by "show configs" commands -#define TSDB_CFG_CTYPE_B_LOG 4U // is a log type configuration -#define TSDB_CFG_CTYPE_B_CLIENT 8U // can be displayed in the client log -#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function -#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password - -#define MAX_FLOAT 100000 -#define MIN_FLOAT 0 - -enum { - TAOS_CFG_CSTATUS_NONE, // not configured - TAOS_CFG_CSTATUS_DEFAULT, // use system default value - TAOS_CFG_CSTATUS_FILE, // configured from file - TAOS_CFG_CSTATUS_OPTION, // configured by taos_options function - TAOS_CFG_CSTATUS_ARG, // configured by program argument -}; - -enum { - TAOS_CFG_VTYPE_INT8, - TAOS_CFG_VTYPE_INT16, - TAOS_CFG_VTYPE_INT32, - TAOS_CFG_VTYPE_UINT16, - TAOS_CFG_VTYPE_FLOAT, - TAOS_CFG_VTYPE_STRING, - TAOS_CFG_VTYPE_IPSTR, - TAOS_CFG_VTYPE_DIRECTORY, - TAOS_CFG_VTYPE_DATA_DIRCTORY, - TAOS_CFG_VTYPE_DOUBLE, -}; - -enum { - TAOS_CFG_UTYPE_NONE, - TAOS_CFG_UTYPE_PERCENT, - TAOS_CFG_UTYPE_GB, - TAOS_CFG_UTYPE_MB, - TAOS_CFG_UTYPE_BYTE, - TAOS_CFG_UTYPE_SECOND, - TAOS_CFG_UTYPE_MS -}; - -typedef struct { - char * option; - void * ptr; - float minValue; - float maxValue; - int8_t cfgType; - int8_t cfgStatus; - int8_t unitType; - int8_t valType; - int32_t ptrLength; -} SGlobalCfg; - -extern SGlobalCfg tsGlobalConfig[]; -extern int32_t tsGlobalConfigNum; -extern char * tsCfgStatusStr[]; - -void taosReadGlobalLogCfg(); -int32_t taosReadCfgFromFile(); -void taosPrintCfg(); -void taosDumpGlobalCfg(); - -void taosAddConfigOption(SGlobalCfg cfg); -SGlobalCfg *taosGetConfigOption(const char *option); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_CONFIG_H*/ diff --git a/include/util/tdef.h b/include/util/tdef.h index a4c333a4d296d4130016d39aad6cd1550dde8ca1..d0f2b77f1fa92ee67ad2a2dce69c234be3b13cf7 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #ifndef _TD_UTIL_DEF_H #define _TD_UTIL_DEF_H @@ -108,35 +110,52 @@ do { \ (src) = (void *)((char *)src + sizeof(type));\ } while(0) +typedef enum EOperatorType { + // arithmetic operator + OP_TYPE_ADD = 1, + OP_TYPE_SUB, + OP_TYPE_MULTI, + OP_TYPE_DIV, + OP_TYPE_MOD, + + // bit operator + OP_TYPE_BIT_AND, + OP_TYPE_BIT_OR, + + // comparison operator + OP_TYPE_GREATER_THAN, + OP_TYPE_GREATER_EQUAL, + OP_TYPE_LOWER_THAN, + OP_TYPE_LOWER_EQUAL, + OP_TYPE_EQUAL, + OP_TYPE_NOT_EQUAL, + OP_TYPE_IN, + OP_TYPE_NOT_IN, + OP_TYPE_LIKE, + OP_TYPE_NOT_LIKE, + OP_TYPE_MATCH, + OP_TYPE_NMATCH, + OP_TYPE_IS_NULL, + OP_TYPE_IS_NOT_NULL, + OP_TYPE_IS_TRUE, + OP_TYPE_IS_FALSE, + OP_TYPE_IS_UNKNOWN, + OP_TYPE_IS_NOT_TRUE, + OP_TYPE_IS_NOT_FALSE, + OP_TYPE_IS_NOT_UNKNOWN, + + // json operator + OP_TYPE_JSON_GET_VALUE, + OP_TYPE_JSON_CONTAINS +} EOperatorType; + + +typedef enum ELogicConditionType { + LOGIC_COND_TYPE_AND, + LOGIC_COND_TYPE_OR, + LOGIC_COND_TYPE_NOT, +} ELogicConditionType; -// TODO: check if below is necessary -#define TSDB_RELATION_INVALID 0 -#define TSDB_RELATION_LESS 1 -#define TSDB_RELATION_GREATER 2 -#define TSDB_RELATION_EQUAL 3 -#define TSDB_RELATION_LESS_EQUAL 4 -#define TSDB_RELATION_GREATER_EQUAL 5 -#define TSDB_RELATION_NOT_EQUAL 6 -#define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_NOT_LIKE 8 -#define TSDB_RELATION_ISNULL 9 -#define TSDB_RELATION_NOTNULL 10 -#define TSDB_RELATION_IN 11 -#define TSDB_RELATION_NOT_IN 12 - -#define TSDB_RELATION_AND 13 -#define TSDB_RELATION_OR 14 -#define TSDB_RELATION_NOT 15 - -#define TSDB_RELATION_MATCH 16 -#define TSDB_RELATION_NMATCH 17 - -#define TSDB_BINARY_OP_ADD 4000 -#define TSDB_BINARY_OP_SUBTRACT 4001 -#define TSDB_BINARY_OP_MULTIPLY 4002 -#define TSDB_BINARY_OP_DIVIDE 4003 -#define TSDB_BINARY_OP_REMAINDER 4004 -#define TSDB_BINARY_OP_CONCAT 4005 #define FUNCTION_CEIL 4500 #define FUNCTION_FLOOR 4501 @@ -148,9 +167,6 @@ do { \ #define FUNCTION_LTRIM 4802 #define FUNCTION_RTRIM 4803 -#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) -#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) - #define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 @@ -182,6 +198,7 @@ do { \ #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_CONSUMER_GROUP_LEN 192 #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE @@ -335,9 +352,6 @@ do { \ #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode - #define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default diff --git a/include/util/tlog.h b/include/util/tlog.h index 26a54173201c209c24461a35a9c4e462742543e0..166e18650890b86a4ddd584ce23ed991e5fcb245 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -22,8 +22,8 @@ extern "C" { #endif -// log -extern int8_t tsAsyncLog; +extern bool tsLogInited; +extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; @@ -32,9 +32,6 @@ extern int32_t mDebugFlag; extern int32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; -extern int32_t httpDebugFlag; -extern int32_t mqttDebugFlag; -extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t qDebugFlag; @@ -42,23 +39,23 @@ extern int32_t wDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; -extern int32_t cqDebugFlag; -extern int32_t debugFlag; - -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U +extern int32_t fsDebugFlag; +#define DEBUG_FATAL 1U +#define DEBUG_ERROR DEBUG_FATAL +#define DEBUG_WARN 2U +#define DEBUG_INFO DEBUG_WARN +#define DEBUG_DEBUG 4U +#define DEBUG_TRACE 8U +#define DEBUG_DUMP 16U #define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U +#define DEBUG_FILE 128U -int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles); +int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); void taosResetLog(); +void taosSetAllDebugFlag(int32_t flag); +void taosDumpData(unsigned char *msg, int32_t len); void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) #ifdef __GNUC__ @@ -72,8 +69,6 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . #endif ; -void taosDumpData(unsigned char *msg, int32_t len); - #ifdef __cplusplus } #endif diff --git a/include/util/tnote.h b/include/util/tnote.h deleted file mode 100644 index e613ec7e4130312808e99dc5ed1f44298031ea1c..0000000000000000000000000000000000000000 --- a/include/util/tnote.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_NOTE_H -#define _TD_UTIL_NOTE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define MAX_NOTE_LINE_SIZE 66000 -#define NOTE_FILE_NAME_LEN 300 - -typedef struct { - int32_t fileNum; - int32_t maxLines; - int32_t lines; - int32_t flag; - int32_t fd; - int32_t openInProgress; - char name[NOTE_FILE_NAME_LEN]; - pthread_mutex_t mutex; -} SNoteObj; - -extern SNoteObj tsHttpNote; -extern SNoteObj tsTscNote; -extern SNoteObj tsInfoNote; - -int32_t taosInitNotes(); -void taosNotePrint(SNoteObj* pNote, const char* const format, ...); -void taosNotePrintBuffer(SNoteObj* pNote, char* buffer, int32_t len); - -#define nPrintHttp(...) \ - if (tsHttpEnableRecordSql) { \ - taosNotePrint(&tsHttpNote, __VA_ARGS__); \ - } - -#define nPrintTsc(...) \ - if (tsTscEnableRecordSql) { \ - taosNotePrint(&tsTscNote, __VA_ARGS__); \ - } - -#define nInfo(buffer, len) \ - if (tscEmbeddedInUtil == 1) { \ - taosNotePrintBuffer(&tsInfoNote, buffer, len); \ - } - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_NOTE_H*/ diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 89ec6cd8d9d000bd98effc8804c52be8ce48c740..01d70c7d02d65fe4340328297b8057bd6a2731f0 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -26,7 +26,6 @@ typedef void *tmr_h; typedef void (*TAOS_TMR_CALLBACK)(void *, void *); extern int taosTmrThreads; -extern uint32_t tsMaxTmrCtrl; #define MSECONDS_PER_TICK 5 diff --git a/include/util/version.h b/include/util/version.h new file mode 100644 index 0000000000000000000000000000000000000000..01efecc59d407df3d79c4ea61b2924aace7c3fa2 --- /dev/null +++ b/include/util/version.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_VERSION_H +#define _TD_UTIL_VERSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern char version[]; +extern char compatible_version[]; +extern char gitinfo[]; +extern char gitinfoOfInternal[]; +extern char buildinfo[]; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_VERSION_H*/ diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 3210c0c6de50889721279cebdf13c67d2e5474e5..c79382555664819f140dd145b0b8e704668a6881 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( taos INTERFACE api - PRIVATE os util common transport parser planner catalog scheduler function qcom + PRIVATE os util common transport parser planner catalog scheduler function qcom config ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d7574466cdfbe15ea38145c7dd3a9d0a3895e9bb..60eb0200a7fb240af8a49f80ad35eb67e5e0d298 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,17 +20,19 @@ extern "C" { #endif -#include "taos.h" #include "common.h" -#include "tmsg.h" +#include "parser.h" +#include "query.h" +#include "taos.h" #include "tdef.h" #include "tep.h" #include "thash.h" #include "tlist.h" +#include "tmsg.h" #include "tmsgtype.h" #include "trpc.h" -#include "query.h" -#include "parser.h" + +#include "config.h" #define CHECK_CODE_GOTO(expr, label) \ do { \ @@ -46,12 +48,12 @@ extern "C" { typedef struct SAppInstInfo SAppInstInfo; typedef struct SHbConnInfo { - void *param; - SClientHbReq *req; + void* param; + SClientHbReq* req; } SHbConnInfo; typedef struct SAppHbMgr { - char *key; + char* key; // statistics int32_t reportCnt; int32_t connKeyCnt; @@ -62,15 +64,13 @@ typedef struct SAppHbMgr { // connection SAppInstInfo* pAppInstInfo; // info - SHashObj* activeInfo; // hash - SHashObj* connInfo; // hash + SHashObj* activeInfo; // hash + SHashObj* connInfo; // hash } SAppHbMgr; +typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); -typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp); - -typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req); - +typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef struct SClientHbMgr { int8_t inited; @@ -83,63 +83,62 @@ typedef struct SClientHbMgr { FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; } SClientHbMgr; - typedef struct SQueryExecMetric { - int64_t start; // start timestamp - int64_t parsed; // start to parse - int64_t send; // start to send to server - int64_t rsp; // receive response from server + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server } SQueryExecMetric; typedef struct SInstanceSummary { - uint64_t numOfInsertsReq; - uint64_t numOfInsertRows; - uint64_t insertElapsedTime; - uint64_t insertBytes; // submit to tsdb since launched. - - uint64_t fetchBytes; - uint64_t queryElapsedTime; - uint64_t numOfSlowQueries; - uint64_t totalRequests; - uint64_t currentRequests; // the number of SRequestObj + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. + + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj } SInstanceSummary; typedef struct SHeartBeatInfo { - void *pTimer; // timer, used to send request msg to mnode + void* pTimer; // timer, used to send request msg to mnode } SHeartBeatInfo; struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; - SList *pConnList; // STscObj linked list - int64_t clusterId; - void *pTransporter; - struct SAppHbMgr *pAppHbMgr; + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; + SList* pConnList; // STscObj linked list + int64_t clusterId; + void* pTransporter; + struct SAppHbMgr* pAppHbMgr; }; typedef struct SAppInfo { int64_t startTime; char appName[TSDB_APP_NAME_LEN]; - char *ep; + char* ep; int32_t pid; int32_t numOfThreads; - SHashObj *pInstMap; + SHashObj* pInstMap; pthread_mutex_t mutex; } SAppInfo; typedef struct STscObj { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_FNAME_LEN]; - char ver[128]; - int32_t acctId; - uint32_t connId; - int32_t connType; - uint64_t id; // ref ID returned by taosAddRef - pthread_mutex_t mutex; // used to protect the operation on db - int32_t numOfReqs; // number of sqlObj bound to this connection - SAppInstInfo *pAppInfo; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char ver[128]; + int32_t acctId; + uint32_t connId; + int32_t connType; + uint64_t id; // ref ID returned by taosAddRef + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj bound to this connection + SAppInstInfo* pAppInfo; } STscObj; typedef struct SMqConsumer { @@ -147,49 +146,49 @@ typedef struct SMqConsumer { } SMqConsumer; typedef struct SReqResultInfo { - const char *pRspMsg; - const char *pData; - TAOS_FIELD *fields; - uint32_t numOfCols; - int32_t *length; - TAOS_ROW row; - char **pCol; - uint32_t numOfRows; - uint64_t totalRows; - uint32_t current; - bool completed; + const char* pRspMsg; + const char* pData; + TAOS_FIELD* fields; + uint32_t numOfCols; + int32_t* length; + TAOS_ROW row; + char** pCol; + uint32_t numOfRows; + uint64_t totalRows; + uint32_t current; + bool completed; } SReqResultInfo; typedef struct SShowReqInfo { - int64_t execId; // showId/queryId - int32_t vgId; - SArray *pArray; // SArray - int32_t currentIndex; // current accessed vgroup index. + int64_t execId; // showId/queryId + int32_t vgId; + SArray* pArray; // SArray + int32_t currentIndex; // current accessed vgroup index. } SShowReqInfo; typedef struct SRequestSendRecvBody { - tsem_t rspSem; // not used now + tsem_t rspSem; // not used now void* fp; - SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; - struct SSchJob *pQueryJob; // query job, created according to sql query DAG. - struct SQueryDag *pDag; // the query dag, generated according to the sql statement. + struct SSchJob* pQueryJob; // query job, created according to sql query DAG. + struct SQueryDag* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; -#define ERROR_MSG_BUF_DEFAULT_SIZE 512 +#define ERROR_MSG_BUF_DEFAULT_SIZE 512 typedef struct SRequestObj { - uint64_t requestId; - int32_t type; // request type - STscObj *pTscObj; - char *sqlstr; // sql string - int32_t sqlLen; - int64_t self; - char *msgBuf; - void *pInfo; // sql parse info, generated by parser module - int32_t code; - SQueryExecMetric metric; + uint64_t requestId; + int32_t type; // request type + STscObj* pTscObj; + char* sqlstr; // sql string + int32_t sqlLen; + int64_t self; + char* msgBuf; + void* pInfo; // sql parse info, generated by parser module + int32_t code; + SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -198,51 +197,52 @@ extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); -int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); +int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int taos_init(); +int taos_init(); -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo); -void destroyTscObj(void*pObj); +void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); +void destroyTscObj(void* pObj); uint64_t generateRequestId(); -void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -char *getDbOfConnection(STscObj* pObj); +char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); -int taos_options_imp(TSDB_OPTION option, const char *str); +int taos_options_imp(TSDB_OPTION option, const char* str); -void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); +void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); +TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, + uint16_t port); -void *doFetchRow(SRequestObj* pRequest); +void* doFetchRow(SRequestObj* pRequest); -void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); - -int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest); +int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery); -// --- heartbeat +// --- heartbeat // global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level -SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); -void appHbMgrCleanup(void); +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); +void appHbMgrCleanup(void); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); @@ -254,6 +254,12 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v void hbMgrInitMqHbRspHandle(); +// config +int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); +int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); + +extern SConfig *tscCfg; + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientCfg.c b/source/client/src/clientCfg.c new file mode 100644 index 0000000000000000000000000000000000000000..d1dafbed5c9a3fd0726a813711cef4da598dbb11 --- /dev/null +++ b/source/client/src/clientCfg.c @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "clientInt.h" +#include "ulog.h" + +// todo refact +SConfig *tscCfg; + +static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char cfgDir[PATH_MAX] = {0}; + char cfgFile[PATH_MAX + 100] = {0}; + + taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); + + if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgFile) != 0) { + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgDir) != 0) { + uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); + return -1; + } + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; + } + + return 0; +} + +static int32_t tscAddLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; + return 0; +} + +static int32_t tscSetLogCfg(SConfig *pCfg) { + osSetLogDir(cfgGetItem(pCfg, "logDir")->str); + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + + int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32; + taosSetAllDebugFlag(debugFlag); + return 0; +} + +int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { + if (tsLogInited) return 0; + + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; + + if (tscAddLogCfg(pCfg) != 0) { + printf("failed to add log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (tscSetLogCfg(pCfg) != 0) { + printf("failed to set log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + const int32_t maxLogFileNum = 10; + if (taosInitLog("taoslog", maxLogFileNum) != 0) { + printf("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + cfgDumpCfg(pCfg); + cfgCleanup(pCfg); + return 0; +} + +static int32_t tscAddEpCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + if (taosGetFqdn(defaultFqdn) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; + + int32_t defaultServerPort = 6030; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1; + + return 0; +} + +static int32_t tscAddCfg(SConfig *pCfg) { + if (tscAddEpCfg(pCfg) != 0) return -1; + + // if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; + // if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; + // if (cfgAddString(pCfg, "version", version) != 0) return -1; + + // if (cfgAddDir(pCfg, "dataDir", osDataDir()) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; + if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; + if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; + // if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; + // if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcTimer", 300, 100, 3000) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcMaxTime", 600, 100, 7200) != 0) return -1; + if (cfgAddInt32(pCfg, "maxConnections", 50000, 1, 100000) != 0) return -1; + return 0; +} + +int32_t tscCheckCfg(SConfig *pCfg) { + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetCoreDump(enableCore); + + return 0; +} + +SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return NULL; + + if (tscAddCfg(pCfg) != 0) { + uError("failed to init tsc cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load tsc cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (tscCheckCfg(pCfg) != 0) { + uError("failed to check cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + cfgDumpCfg(pCfg); + return pCfg; +} + +int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { + tscCfg = tscInitCfgImp(cfgDir, envFile, apolloUrl); + if (tscCfg == NULL) return -1; + + return 0; +} \ No newline at end of file diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index cb73701bfa389ede7ed8073728acd7d418a369c5..f07048efbc0ed151ca63dbe782626d7078105a37 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,33 +13,30 @@ * along with this program. If not, see . */ -#include "os.h" #include "catalog.h" #include "clientInt.h" #include "clientLog.h" +#include "os.h" #include "query.h" #include "scheduler.h" -#include "tmsg.h" #include "tcache.h" -#include "tconfig.h" #include "tglobal.h" -#include "tnote.h" +#include "tmsg.h" #include "tref.h" #include "trpc.h" #include "ttime.h" -#include "ttimezone.h" #define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 +#define TSC_VAR_RELEASED 0 -SAppInfo appInfo; -int32_t clientReqRefPool = -1; -int32_t clientConnRefPool = -1; +SAppInfo appInfo; +int32_t clientReqRefPool = -1; +int32_t clientConnRefPool = -1; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -volatile int32_t tscInitRes = 0; +volatile int32_t tscInitRes = 0; -static void registerRequest(SRequestObj* pRequest) { +static void registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = (STscObj *)taosAcquireRef(clientConnRefPool, pRequest->pTscObj->id); assert(pTscObj != NULL); @@ -53,69 +50,58 @@ static void registerRequest(SRequestObj* pRequest) { int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1); int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1); - tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%"PRIx64, pRequest->self, - pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 + ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, + pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } } -static void deregisterRequest(SRequestObj* pRequest) { +static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj* pTscObj = pRequest->pTscObj; - SInstanceSummary* pActivity = &pTscObj->pAppInfo->summary; + STscObj * pTscObj = pRequest->pTscObj; + SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampMs() - pRequest->metric.start; - tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id, - pRequest->requestId, duration, num, currentInst); + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 + " ms, current:%d, app current:%d", + pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst); taosReleaseRef(clientConnRefPool, pTscObj->id); } -static void tscInitLogFile() { - taosReadGlobalLogCfg(); - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } -} // todo close the transporter properly -void closeTransporter(STscObj* pTscObj) { +void closeTransporter(STscObj *pTscObj) { if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { return; } - tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } // TODO refactor -void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { +void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; - rpcInit.sessions = tsMaxConnections; + rpcInit.pfp = persistConnForSpecificMsg; + rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; - rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.idleTime = cfgGetItem(tscCfg, "shellActivityTimer")->i32 * 1000; rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.secret = (char *)auth; - void* pDnodeConn = rpcOpen(&rpcInit); + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); return NULL; @@ -130,12 +116,12 @@ void destroyTscObj(void *pObj) { SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); pthread_mutex_destroy(&pTscObj->mutex); tfree(pTscObj); } -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo) { +void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -153,11 +139,11 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI pthread_mutex_init(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - tscDebug("connObj created, 0x%"PRIx64, pObj->id); + tscDebug("connObj created, 0x%" PRIx64, pObj->id); return pObj; } -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { +void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); @@ -166,20 +152,20 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty return NULL; } - pRequest->requestId = generateRequestId(); + pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampMs(); - pRequest->type = type; - pRequest->pTscObj = pObj; - pRequest->body.fp = fp; // not used it yet - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; // not used it yet + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); return pRequest; } -static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { +static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { tfree(pResInfo->pRspMsg); tfree(pResInfo->length); tfree(pResInfo->row); @@ -187,9 +173,9 @@ static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { tfree(pResInfo->fields); } -static void doDestroyRequest(void* p) { +static void doDestroyRequest(void *p) { assert(p != NULL); - SRequestObj* pRequest = (SRequestObj*)p; + SRequestObj *pRequest = (SRequestObj *)p; assert(RID_VALID(pRequest->self)); @@ -208,7 +194,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest); } -void destroyRequest(SRequestObj* pRequest) { +void destroyRequest(SRequestObj *pRequest) { if (pRequest == NULL) { return; } @@ -225,41 +211,46 @@ void taos_init_imp(void) { srand(taosGetTimestampSec()); deltaToUtcInitOnce(); - taosInitGlobalCfg(); - taosReadCfgFromFile(); - tscInitLogFile(); - if (taosCheckAndPrintCfg()) { + if (tscInitLog(configDir, NULL, NULL) != 0) { + tscInitRes = -1; + return; + } + + if (tscInitCfg(configDir, NULL, NULL) != 0) { tscInitRes = -1; return; } - taosInitNotes(); initMsgHandleFp(); initQueryModuleMsgHandle(); - rpcInit(); + SRpcCfg rpcCfg = {0}; + rpcCfg.rpcTimer = cfgGetItem(tscCfg, "rpcTimer")->i32; + rpcCfg.rpcMaxTime = cfgGetItem(tscCfg, "rpcMaxTime")->i32; + rpcCfg.sver = 30000000; + rpcInit(&rpcCfg); SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; catalogInit(&cfg); SSchedulerCfg scfg = {.maxJobNum = 100}; schedulerInit(&scfg); - tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + tscDebug("starting to initialize TAOS driver"); taosSetCoreDump(true); initTaskQueue(); clientConnRefPool = taosOpenRef(200, destroyTscObj); - clientReqRefPool = taosOpenRef(40960, doDestroyRequest); + clientReqRefPool = taosOpenRef(40960, doDestroyRequest); taosGetAppName(appInfo.appName, NULL); pthread_mutex_init(&appInfo.mutex, NULL); - appInfo.pid = taosGetPId(); + appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); - appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tscDebug("client is initialized successfully"); } @@ -269,6 +260,7 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { +#if 0 SGlobalCfg *cfg = NULL; switch (option) { @@ -281,7 +273,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set config file directory:%s", str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -296,7 +289,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); } break; @@ -305,7 +299,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { + if (len == 0 || len > TD_LOCALE_LEN) { tscInfo("Invalid locale:%s, use default", str); return -1; } @@ -313,8 +307,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { char sep = '.'; - if (strlen(tsLocale) == 0) { // locale does not set yet - char* defaultLocale = setlocale(LC_CTYPE, ""); + if (strlen(tsLocale) == 0) { // locale does not set yet + char *defaultLocale = setlocale(LC_CTYPE, ""); // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. // The launch of current system will abort soon. @@ -323,21 +317,21 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return -1; } - tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN); } // set the user specified locale char *locale = setlocale(LC_CTYPE, str); - if (locale != NULL) { // failed to set the user specified locale + if (locale != NULL) { // failed to set the user specified locale tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale + } else { // set the user specified locale failed, use default LC_CTYPE as current locale locale = setlocale(LC_CTYPE, tsLocale); tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); } - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + tstrncpy(tsLocale, locale, TD_LOCALE_LEN); char *charset = strrchr(tsLocale, sep); if (charset != NULL) { @@ -352,7 +346,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset changed from %s to %s", tsCharset, charset); } - tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, charset, TD_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { @@ -360,11 +354,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { } free(charset); - } else { // it may be windows system + } else { // it may be windows system tscInfo("charset remains:%s", tsCharset); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; } @@ -375,7 +370,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { + if (len == 0 || len > TD_LOCALE_LEN) { tscInfo("failed to set charset:%s", str); return -1; } @@ -388,13 +383,14 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset changed from %s to %s", tsCharset, str); } - tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, str, TD_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { tscInfo("charset:%s not valid", str); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -405,12 +401,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); + tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN); tsSetTimeZone(); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -419,7 +416,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscError("Invalid option %d", option); return -1; } - +#endif return 0; } @@ -434,7 +431,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { */ uint64_t generateRequestId() { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { char uid[64] = {0}; @@ -448,9 +445,9 @@ uint64_t generateRequestId() { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2eff353c3947d2fa7f908300b79feadd0c75f93c..c9b2e371a232d1bdaedf84d9f2a94a838d5f8830 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -8,11 +8,10 @@ #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tnote.h" #include "tpagedbuf.h" #include "tref.h" -static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); +static int32_t initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); @@ -81,19 +80,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, } SCorEpSet epSet = {0}; - if (ip) { - if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { - return NULL; - } - - if (port) { - epSet.epSet.eps[0].port = port; - } - } else { - if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { - return NULL; - } - } + initEpSetFromCfg(ip, port, &epSet); char* key = getClusterKey(user, secretEncrypt, ip, port); SAppInstInfo** pInst = NULL; @@ -254,11 +241,9 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { return NULL; } - nPrintTsc("%s", sql) - - SRequestObj* pRequest = NULL; - SQueryNode* pQueryNode = NULL; - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + SRequestObj* pRequest = NULL; + SQueryNode* pQueryNode = NULL; + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -282,32 +267,40 @@ _return: return pRequest; } -int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { - pEpSet->version = 0; +int initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet) { + SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp"); + SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp"); + SConfigItem* pPort = cfgGetItem(tscCfg, "serverPort"); // init mnode ip set SEpSet* mgmtEpSet = &(pEpSet->epSet); mgmtEpSet->numOfEps = 0; mgmtEpSet->inUse = 0; + pEpSet->version = 0; - if (firstEp && firstEp[0] != 0) { - if (strlen(firstEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (ip != NULL) { + taosGetFqdnPortFromEp(ip, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); mgmtEpSet->numOfEps++; - } - - if (secondEp && secondEp[0] != 0) { - if (strlen(secondEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; + if (port) { + mgmtEpSet->eps[0].port = port; + } + } else { + if (pFirst->str[0] != 0) { + if (strlen(pFirst->str) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + taosGetFqdnPortFromEp(pFirst->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); + mgmtEpSet->numOfEps++; + } + if (pSecond->str[0] != 0) { + if (strlen(pSecond->str) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + taosGetFqdnPortFromEp(pSecond->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[1]); + mgmtEpSet->numOfEps++; } - - taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); - mgmtEpSet->numOfEps++; } if (mgmtEpSet->numOfEps == 0) { @@ -370,7 +363,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; pMsgSendInfo->param = pRequest; - SConnectReq connectReq = {0}; STscObj* pObj = pRequest->pTscObj; @@ -398,7 +390,9 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { tfree(pMsgBody->msgInfo.pData); tfree(pMsgBody); } - +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { + return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; +} void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle; assert(pMsg->ahandle != NULL); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b29813be03b3163d0399133c2d12c658eda3ccc5..5bcc287116d2de8a9f713f26cfdbed74c7a52fd9 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -7,6 +7,7 @@ #include "tmsg.h" #include "tglobal.h" #include "catalog.h" +#include "version.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -56,9 +57,7 @@ void taos_cleanup(void) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0) ? port : tsServerPort; - - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { user = TSDB_DEFAULT_USER; } @@ -67,7 +66,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha pass = TSDB_DEFAULT_PASS; } - return taos_connect_internal(ip, user, pass, NULL, db, p); + return taos_connect_internal(ip, user, pass, NULL, db, port); } void taos_close(TAOS* taos) { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 229d3a9ec319472f3ead8f0a1a984d5a800ea842..d3d9fce21097a93ff0404533c010d7cd2c99523e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -24,7 +24,6 @@ #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tnote.h" #include "tpagedbuf.h" #include "tref.h" @@ -45,22 +44,24 @@ struct tmq_topic_vgroup_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - bool auto_commit; + char clientId[256]; + char groupId[256]; + int8_t auto_commit; + int8_t resetOffset; + tmq_commit_cb* commit_cb; /*char* ip;*/ /*uint16_t port;*/ - tmq_commit_cb* commit_cb; }; struct tmq_t { // conf char groupId[256]; char clientId[256]; - bool autoCommit; + int8_t autoCommit; SRWLatch lock; int64_t consumerId; int32_t epoch; + int32_t resetOffsetCfg; int64_t status; tsem_t rspSem; STscObj* pTscObj; @@ -79,7 +80,6 @@ typedef struct { // statistics int64_t pollCnt; // offset - int64_t committedOffset; int64_t currentOffset; // connection info int32_t vgId; @@ -115,21 +115,17 @@ typedef struct { } SMqConsumeCbParam; typedef struct { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t async; - tsem_t rspSem; -} SMqCommitCbParam; - -typedef struct { - tmq_t* tmq; + tmq_t* tmq; + /*SMqClientVg* pVg;*/ + int32_t async; tsem_t rspSem; tmq_resp_err_t rspErr; -} SMqResetOffsetParam; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; return conf; } @@ -157,6 +153,20 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_INVALID; } } + if (strcmp(key, "auto.offset.reset") == 0) { + if (strcmp(value, "none") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; + return TMQ_CONF_OK; + } else if (strcmp(value, "earliest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + return TMQ_CONF_OK; + } else if (strcmp(value, "latest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } return TMQ_CONF_UNKNOWN; } @@ -190,14 +200,12 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { if (pParam->tmq->commit_cb) { pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); } - if (!pParam->async) tsem_post(&pParam->rspSem); - return 0; -} - -int32_t tmqResetOffsetCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqResetOffsetParam* pParam = (SMqResetOffsetParam*)param; - pParam->rspErr = code; - tsem_post(&pParam->rspSem); + if (!pParam->async) + tsem_post(&pParam->rspSem); + else { + tsem_destroy(&pParam->rspSem); + free(param); + } return 0; } @@ -216,6 +224,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs strcpy(pTmq->groupId, conf->groupId); pTmq->autoCommit = conf->auto_commit; pTmq->commit_cb = conf->commit_cb; + pTmq->resetOffsetCfg = conf->resetOffset; tsem_init(&pTmq->rspSem, 0, 0); pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); @@ -223,18 +232,40 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return pTmq; } -tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) { - SRequestObj* pRequest = NULL; +tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { + // TODO: add read write lock + SRequestObj* pRequest = NULL; + tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS; // build msg // send to mnode - SMqCMResetOffsetReq req; - req.num = offsets->cnt; - req.offsets = (SMqOffset*)offsets->elems; + SMqCMCommitOffsetReq req; + SArray* pArray = NULL; + + if (offsets == NULL) { + pArray = taosArrayInit(0, sizeof(SMqOffset)); + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqOffset offset; + strcpy(offset.topicName, pTopic->topicName); + strcpy(offset.cgroup, tmq->groupId); + offset.vgId = pVg->vgId; + offset.offset = pVg->currentOffset; + taosArrayPush(pArray, &offset); + } + } + req.num = pArray->size; + req.offsets = pArray->pData; + } else { + req.num = offsets->cnt; + req.offsets = (SMqOffset*)offsets->elems; + } SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); int32_t tlen = encoder.pos; void* buf = malloc(tlen); if (buf == NULL) { @@ -244,32 +275,41 @@ tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offse tCoderClear(&encoder); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); tCoderClear(&encoder); - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_RESET_OFFSET); + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET); if (pRequest == NULL) { tscError("failed to malloc request"); } - SMqResetOffsetParam param = {0}; - tsem_init(¶m.rspSem, 0, 0); - param.tmq = tmq; + SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + return -1; + } + pParam->tmq = tmq; + tsem_init(&pParam->rspSem, 0, 0); pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->param = ¶m; - sendInfo->fp = tmqResetOffsetCb; + sendInfo->param = pParam; + sendInfo->fp = tmqCommitCb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_wait(¶m.rspSem); - tsem_destroy(¶m.rspSem); + if (!async) { + tsem_wait(&pParam->rspSem); + resp = pParam->rspErr; + } - return param.rspErr; + if (pArray) { + taosArrayDestroy(pArray); + } + + return resp; } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { @@ -641,8 +681,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { // clang-format off SMqClientVg clientVg = { .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, + .currentOffset = pVgEp->offset, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet }; @@ -708,23 +747,51 @@ END: return 0; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, - SMqClientVg* pVg) { +tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { + const SMqOffset* pOffset = &offset->offset; + if (strcmp(pOffset->cgroup, tmq->groupId) != 0) { + return TMQ_RESP_ERR__FAIL; + } + int32_t sz = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) { + int32_t vgSz = taosArrayGetSize(clientTopic->vgs); + for (int32_t j = 0; j < vgSz; j++) { + SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); + if (pVg->vgId == pOffset->vgId) { + pVg->currentOffset = pOffset->offset; + return TMQ_RESP_ERR__SUCCESS; + } + } + } + } + return TMQ_RESP_ERR__FAIL; +} + +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) { + int64_t reqOffset; + if (pVg->currentOffset >= 0) { + reqOffset = pVg->currentOffset; + } else { + if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) { + tscError("unable to poll since no committed offset but reset offset is set to none"); + return NULL; + } + reqOffset = tmq->resetOffsetCfg; + } + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); if (pReq == NULL) { return NULL; } - pReq->reqType = type; + strcpy(pReq->topic, pTopic->topicName); - pReq->blockingTime = blocking_time; - pReq->consumerId = tmq->consumerId; strcpy(pReq->cgroup, tmq->groupId); - if (type == TMQ_REQ_TYPE_COMMIT_ONLY) { - pReq->offset = pVg->currentOffset; - } else { - pReq->offset = pVg->currentOffset + 1; - } + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + pReq->currentOffset = reqOffset; pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); @@ -743,13 +810,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - printf("over1\n"); + /*printf("over1\n");*/ usleep(blocking_time * 1000); return NULL; } SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); if (taosArrayGetSize(pTopic->vgs) == 0) { - printf("over2\n"); + /*printf("over2\n");*/ usleep(blocking_time * 1000); return NULL; } @@ -760,8 +827,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ - int32_t reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY; - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg); if (pReq == NULL) { ASSERT(false); usleep(blocking_time * 1000); @@ -821,6 +887,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { /*return pRequest;*/ } +#if 0 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { if (tmq_topic_vgroup_list != NULL) { // TODO @@ -831,7 +898,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; @@ -858,6 +925,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v return 0; } +#endif void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index ee5109860e52968d487cad9ace18153443974f32..7f6ce81788f66dc8d5d2182b8f86813dbb49940a 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(clientTest clientTests.cpp) TARGET_LINK_LIBRARIES( clientTest - PUBLIC os util common transport parser catalog scheduler function gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config ) ADD_EXECUTABLE(tmqTest tmqTest.cpp) TARGET_LINK_LIBRARIES( tmqTest - PUBLIC os util common transport parser catalog scheduler function gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 5414f17a6247769ba38c84d0dac8c37d8e25168e..ac36179a264ad8a23c6f7431279b36abe5b68458 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -49,7 +49,7 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - taosInitGlobalCfg(); + // taosInitGlobalCfg(); // taos_init(); } diff --git a/source/client/test/tmqTest.cpp b/source/client/test/tmqTest.cpp index 9f8ff7143a822cfe3abb69b1475f884397501436..c34c10d8714fa443bb161333ab622093ec6eebd7 100644 --- a/source/client/test/tmqTest.cpp +++ b/source/client/test/tmqTest.cpp @@ -34,7 +34,7 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - taosInitGlobalCfg(); + // taosInitGlobalCfg(); // taos_init(); } diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 970b6d954f1faa4f1f439f1f415e0d7059852311..3e98cae27923d7c845503e35f8fd0846440fd38e 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -4,7 +4,7 @@ #include "tglobal.h" #include "tlockfree.h" -int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { +int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) { pEp->port = 0; strcpy(pEp->fqdn, ep); @@ -15,7 +15,7 @@ int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { } if (pEp->port == 0) { - pEp->port = tsServerPort; + pEp->port = defaultPort; return -1; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index afba1190babb7fcd1c2e48a1aa80568b7cfca213..b4c9efd77d6a93ea755cfa0902e54eb6a38561e9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -19,38 +19,14 @@ #include "taosdef.h" #include "taoserror.h" #include "tcompare.h" -#include "tconfig.h" #include "tep.h" #include "tglobal.h" -#include "tlocale.h" #include "tlog.h" -#include "ttimezone.h" #include "tutil.h" #include "ulog.h" -// cluster -char tsFirst[TSDB_EP_LEN] = {0}; -char tsSecond[TSDB_EP_LEN] = {0}; -char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; -char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port -uint16_t tsServerPort = 6030; -int32_t tsStatusInterval = 1; // second -int8_t tsEnableTelemetryReporting = 0; -char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsNumOfSupportVnodes = 128; // common -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default -int32_t tsMaxShellConns = 50000; -int32_t tsMaxConnections = 50000; -int32_t tsShellActivityTimer = 3; // second -float tsNumOfThreadsPerCore = 1.0f; -int32_t tsNumOfCommitThreads = 4; -float tsRatioOfQueryCores = 1.0f; -int8_t tsDaylight = 0; -int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; @@ -81,8 +57,6 @@ int32_t tsCompatibleModel = 1; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; -int8_t tsTscEnableRecordSql = 0; - // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp int32_t tsMaxNumOfOrderedResults = 100000; @@ -131,14 +105,6 @@ int8_t tsDeadLockKillQuery = 0; // For backward compatibility bool tsdbForceKeepFile = false; -int32_t tsDiskCfgNum = 0; - -#ifndef _STORAGE -SDiskCfg tsDiskCfg[1]; -#else -SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; -#endif - /* * minimum scale for whole system, millisecond by default * for TSDB_TIME_PRECISION_MILLI: 86400000L @@ -147,30 +113,6 @@ SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; */ int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; -// system info -float tsTotalTmpDirGB = 0; -float tsTotalDataDirGB = 0; -float tsAvailTmpDirectorySpace = 0; -float tsAvailDataDirGB = 0; -float tsUsedDataDirGB = 0; -float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 2.0f; -int32_t tsTotalMemoryMB = 0; -uint32_t tsVersion = 0; - -// -// lossy compress 6 -// -char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty - // can close lossy compress. -// below option can take effect when tsLossyColumns not empty -double tsFPrecision = 1E-8; // float column precision -double tsDPrecision = 1E-16; // double column precision -uint32_t tsMaxRange = 500; // max range -uint32_t tsCurRange = 100; // range -char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR - - int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; void (*monExecuteSQLFp)(char *sql) = NULL; @@ -179,24 +121,9 @@ char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; -void taosSetAllDebugFlag() { - if (debugFlag != 0) { - mDebugFlag = debugFlag; - dDebugFlag = debugFlag; - vDebugFlag = debugFlag; - jniDebugFlag = debugFlag; - qDebugFlag = debugFlag; - rpcDebugFlag = debugFlag; - uDebugFlag = debugFlag; - sDebugFlag = debugFlag; - wDebugFlag = debugFlag; - tsdbDebugFlag = debugFlag; - cqDebugFlag = debugFlag; - uInfo("all debug flag are set to %d", debugFlag); - } -} int32_t taosCfgDynamicOptions(char *msg) { + #if 0 char *option, *value; int32_t olen, vlen; int32_t vint = 0; @@ -266,133 +193,46 @@ int32_t taosCfgDynamicOptions(char *msg) { } } +#endif return false; } -void taosAddDataDir(int index, char *v1, int level, int primary) { - tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); - tsDiskCfg[index].level = level; - tsDiskCfg[index].primary = primary; - uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); -} +// void taosAddDataDir(int index, char *v1, int level, int primary) { +// tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); +// tsDiskCfg[index].level = level; +// tsDiskCfg[index].primary = primary; +// uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); +// } #ifndef _STORAGE -void taosReadDataDirCfg(char *v1, char *v2, char *v3) { - if (tsDiskCfgNum == 1) { - SDiskCfg *cfg = &tsDiskCfg[0]; - uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); - } - taosAddDataDir(0, v1, 0, 1); - tsDiskCfgNum = 1; -} - -void taosPrintDataDirCfg() { - for (int i = 0; i < tsDiskCfgNum; ++i) { - SDiskCfg *cfg = &tsDiskCfg[i]; - uInfo(" dataDir: %s", cfg->dir); - } -} +// void taosReadDataDirCfg(char *v1, char *v2, char *v3) { +// if (tsDiskCfgNum == 1) { +// SDiskCfg *cfg = &tsDiskCfg[0]; +// uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); +// } +// taosAddDataDir(0, v1, 0, 1); +// tsDiskCfgNum = 1; +// } + +// void taosPrintDataDirCfg() { +// for (int i = 0; i < tsDiskCfgNum; ++i) { +// SDiskCfg *cfg = &tsDiskCfg[i]; +// uInfo(" dataDir: %s", cfg->dir); +// } +// } #endif -static void taosCheckDataDirCfg() { - if (tsDiskCfgNum <= 0) { - taosAddDataDir(0, tsDataDir, 0, 1); - tsDiskCfgNum = 1; - uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); - } -} + static void doInitGlobalConfig(void) { osInit(); srand(taosSafeRand()); - +#if 0 SGlobalCfg cfg = {0}; - // ip address - cfg.option = "firstEp"; - cfg.ptr = tsFirst; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "secondEp"; - cfg.ptr = tsSecond; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fqdn"; - cfg.ptr = tsLocalFqdn; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FQDN_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // port - cfg.option = "serverPort"; - cfg.ptr = &tsServerPort; - cfg.valType = TAOS_CFG_VTYPE_UINT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65056; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "supportVnodes"; - cfg.ptr = &tsNumOfSupportVnodes; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // directory - cfg.option = "configDir"; - cfg.ptr = configDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "logDir"; - cfg.ptr = tsLogDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "scriptDir"; - cfg.ptr = tsScriptDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "dataDir"; - cfg.ptr = tsDataDir; + cfg.ptr = osDataDir(); cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; @@ -401,36 +241,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // dnode configs - cfg.option = "numOfThreadsPerCore"; - cfg.ptr = &tsNumOfThreadsPerCore; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "numOfCommitThreads"; - cfg.ptr = &tsNumOfCommitThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 100; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "ratioOfQueryCores"; - cfg.ptr = &tsRatioOfQueryCores; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0.0f; - cfg.maxValue = 2.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "maxNumOfDistinctRes"; cfg.ptr = &tsMaxNumOfDistinctResults; @@ -442,77 +252,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "telemetryReporting"; - cfg.ptr = &tsEnableTelemetryReporting; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // timer - cfg.option = "maxTmrCtrl"; - cfg.ptr = &tsMaxTmrCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 8; - cfg.maxValue = 2048; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcTimer"; - cfg.ptr = &tsRpcTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 3000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "rpcForceTcp"; - cfg.ptr = &tsRpcForceTcp; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcMaxTime"; - cfg.ptr = &tsRpcMaxTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 7200; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "statusInterval"; - cfg.ptr = &tsStatusInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "shellActivityTimer"; - cfg.ptr = &tsShellActivityTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 120; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -653,87 +392,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // locale & charset - cfg.option = "timezone"; - cfg.ptr = tsTimezone; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_TIMEZONE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "locale"; - cfg.ptr = tsLocale; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "charset"; - cfg.ptr = tsCharset; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // connect configs - cfg.option = "maxShellConns"; - cfg.ptr = &tsMaxShellConns; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 50000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxConnections"; - cfg.ptr = &tsMaxConnections; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "minimalLogDirGB"; - cfg.ptr = &tsMinimalLogDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalTmpDirGB"; - cfg.ptr = &tsReservedTmpDirectorySpace; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - cfg.option = "minimalDataDirGB"; - cfg.ptr = &tsMinimalDataDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); cfg.option = "slaveQuery"; cfg.ptr = &tsEnableSlaveQuery; @@ -745,237 +404,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // debug flag - cfg.option = "numOfLogLines"; - cfg.ptr = &tsNumOfLogLines; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1000; - cfg.maxValue = 2000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "logKeepDays"; - cfg.ptr = &tsLogKeepDays; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = -365000; - cfg.maxValue = 365000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "asyncLog"; - cfg.ptr = &tsAsyncLog; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "debugFlag"; - cfg.ptr = &debugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mDebugFlag"; - cfg.ptr = &mDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "dDebugFlag"; - cfg.ptr = &dDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "sDebugFlag"; - cfg.ptr = &sDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "wDebugFlag"; - cfg.ptr = &wDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcDebugFlag"; - cfg.ptr = &rpcDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tmrDebugFlag"; - cfg.ptr = &tmrDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cDebugFlag"; - cfg.ptr = &cDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "jniDebugFlag"; - cfg.ptr = &jniDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "uDebugFlag"; - cfg.ptr = &uDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "qDebugFlag"; - cfg.ptr = &qDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "vDebugFlag"; - cfg.ptr = &vDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tsdbDebugFlag"; - cfg.ptr = &tsdbDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cqDebugFlag"; - cfg.ptr = &cqDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "enableRecordSql"; - cfg.ptr = &tsTscEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "enableCoreFile"; - cfg.ptr = &tsEnableCoreFile; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // version info - cfg.option = "gitinfo"; - cfg.ptr = gitinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "gitinfoOfInternal"; - cfg.ptr = gitinfoOfInternal; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "buildinfo"; - cfg.ptr = buildinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "version"; - cfg.ptr = version; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "maxBinaryDisplayWidth"; cfg.ptr = &tsMaxBinaryDisplayWidth; @@ -988,7 +416,7 @@ static void doInitGlobalConfig(void) { taosAddConfigOption(cfg); cfg.option = "tempDir"; - cfg.ptr = tsTempDir; + cfg.ptr = osTempDir(); cfg.valType = TAOS_CFG_VTYPE_STRING; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; @@ -1025,7 +453,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_DOUBLE; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; + cfg.maxValue = 100000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; @@ -1035,8 +463,8 @@ static void doInitGlobalConfig(void) { cfg.ptr = &dPrecision; cfg.valType = TAOS_CFG_VTYPE_DOUBLE; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; + cfg.minValue = 100000; + cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); @@ -1064,65 +492,8 @@ static void doInitGlobalConfig(void) { #else // assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); #endif -} - -void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -int32_t taosCheckAndPrintCfg() { - SEp ep = {0}; - if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - taosSetAllDebugFlag(); - } - - if (tsLocalFqdn[0] == 0) { - taosGetFqdn(tsLocalFqdn); - } - - snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); - uInfo("localEp is: %s", tsLocalEp); - - if (tsFirst[0] == 0) { - strcpy(tsFirst, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsFirst, &ep); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port); - } - - if (tsSecond[0] == 0) { - strcpy(tsSecond, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsSecond, &ep); - snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port); - } - - taosCheckDataDirCfg(); - - if (taosDirExist(tsTempDir) != 0) { - return -1; - } - - taosGetSystemInfo(); - - tsSetLocale(); - - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { - tsSetTimeZone(); - } - - if (tsNumOfCores <= 0) { - tsNumOfCores = 1; - } - - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; - } - - uInfo(" check global cfg completed"); - uInfo("=================================="); - taosPrintCfg(); - - return 0; +#endif } /* diff --git a/source/common/src/tlocale.c b/source/common/src/tlocale.c deleted file mode 100644 index a98a46b28a658924042b35df93d17db18b2c2f54..0000000000000000000000000000000000000000 --- a/source/common/src/tlocale.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tconfig.h" -#include "tutil.h" - -/** - * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of - * both the TDengine Server and the Client may be interrupted. - * - * In case that the setLocale failed to be executed, the right charset needs to be set. - */ -void tsSetLocale() { - char *locale = setlocale(LC_CTYPE, tsLocale); - - // default locale or user specified locale is not valid, abort launch - if (locale == NULL) { - uError("Invalid locale:%s, please set the valid locale in config file", tsLocale); - } - - if (strlen(tsCharset) == 0) { - uError("failed to get charset, please set the valid charset in config file"); - exit(-1); - } - - if (!taosValidateEncodec(tsCharset)) { - uError("Invalid charset:%s, please set the valid charset in config file", tsCharset); - exit(-1); - } -} \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 46feab779144f89272f977f58a7ae463d8063250..8101c18ebf43ce78fb119c77d8eeca6aea6fbd6e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -285,7 +285,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); @@ -330,7 +330,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { } void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); + buf = taosDecodeFixedI64(buf, &(pReq->ver)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); @@ -380,7 +380,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray)); for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i); @@ -393,7 +393,7 @@ int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { uint32_t nsize = 0; - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeFixedU32(buf, &nsize); pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq)); for (size_t i = 0; i < nsize; i++) { @@ -407,14 +407,14 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU8(buf, pReq->type); return tlen; } void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeString(buf, &pReq->name); buf = taosDecodeFixedU8(buf, &pReq->type); return buf; @@ -1393,7 +1393,7 @@ int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1; - if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->uid) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1407,7 +1407,7 @@ int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1; - if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->uid) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); @@ -1468,7 +1468,7 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; @@ -1518,7 +1518,7 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; @@ -1661,7 +1661,7 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->dbId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1; @@ -1684,7 +1684,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->dbId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1; @@ -2093,7 +2093,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; @@ -2133,7 +2133,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; @@ -2171,7 +2171,7 @@ int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; tEndEncode(&encoder); @@ -2187,7 +2187,7 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; tEndDecode(&decoder); @@ -2356,35 +2356,7 @@ int32_t tDecodeSMqOffset(SCoder *decoder, SMqOffset *pOffset) { return 0; } -int32_t tEncodeSMqVgOffsets(SCoder *encoder, const SMqVgOffsets *pOffsets) { - if (tStartEncode(encoder) < 0) return -1; - if (tEncodeI32(encoder, pOffsets->vgId) < 0) return -1; - int32_t sz = taosArrayGetSize(pOffsets->offsets); - if (tEncodeI32(encoder, sz) < 0) return -1; - for (int32_t i = 0; i < sz; i++) { - SMqOffset *offset = taosArrayGet(pOffsets->offsets, i); - if (tEncodeSMqOffset(encoder, offset) < 0) return -1; - } - tEndEncode(encoder); - return encoder->pos; -} - -int32_t tDecodeSMqVgOffsets(SCoder *decoder, SMqVgOffsets *pOffsets) { - int32_t sz; - if (tStartDecode(decoder) < 0) return -1; - if (tDecodeI32(decoder, &pOffsets->vgId) < 0) return -1; - if (tDecodeI32(decoder, &sz) < 0) return -1; - pOffsets->offsets = taosArrayInit(sz, sizeof(SMqOffset)); - for (int32_t i = 0; i < sz; i++) { - SMqOffset offset; - if (tDecodeSMqOffset(decoder, &offset) < 0) return -1; - taosArrayPush(pOffsets->offsets, &offset); - } - tEndDecode(decoder); - return 0; -} - -int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *pReq) { +int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq *pReq) { if (tStartEncode(encoder) < 0) return -1; if (tEncodeI32(encoder, pReq->num) < 0) return -1; for (int32_t i = 0; i < pReq->num; i++) { @@ -2394,7 +2366,7 @@ int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *p return encoder->pos; } -int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { +int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); @@ -2405,23 +2377,3 @@ int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { tEndDecode(decoder); return 0; } - -#if 0 -int32_t tEncodeSMqMVResetOffsetReq(SCoder *encoder, const SMqMVResetOffsetReq *pReq) { - if (tEncodeI64(encoder, pReq->leftForVer) < 0) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tEncodeSMqOffset(encoder, &pReq->offsets[i]); - } - return encoder->pos; -} - -int32_t tDecodeSMqMVResetOffsetReq(SCoder *decoder, SMqMVResetOffsetReq *pReq) { - if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); - if (pReq->offsets == NULL) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tDecodeSMqOffset(decoder, &pReq->offsets[i]); - } - return 0; -} -#endif diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f3deb84ccf909e62ece7d36566be36b9c7e429ea..f6892b26bdb0fc167548d1d95287e5f21bbf6678 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -31,7 +31,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil } assert(src->filterstr == 0 || src->filterstr == 1); - assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0)); return pFilter; } diff --git a/source/common/src/ttimezone.c b/source/common/src/ttimezone.c deleted file mode 100644 index c45e39c20d458bff58d35c8e76848051617da8eb..0000000000000000000000000000000000000000 --- a/source/common/src/ttimezone.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tconfig.h" -#include "tutil.h" - -// TODO refactor to set the tz value through parameter -void tsSetTimeZone() { - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone != NULL) { - uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]); - } - -#ifdef WINDOWS - char winStr[TSDB_LOCALE_LEN * 2]; - sprintf(winStr, "TZ=%s", tsTimezone); - putenv(winStr); -#else - setenv("TZ", tsTimezone, 1); -#endif - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char **tzname = _tzname; -#endif -#endif - - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - tz += daylight; - - /* - * format: - * (CST, +0800) - * (BST, +0100) - */ - sprintf(tsTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - tsDaylight = daylight; - - uInfo("timezone format changed to %s", tsTimezone); -} diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 6d57992c35bab4b8cea04888f1cf78ab9417bcb2..3265ea55476025e14357e0c7b021d93416faf763 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -23,7 +23,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { pTSBuf->autoDelete = autoDelete; - taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path); + taosGetTmpfilePath(osTempDir(), "join", pTSBuf->path); pTSBuf->f = fopen(pTSBuf->path, "wb+"); if (pTSBuf->f == NULL) { free(pTSBuf); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 1cab413c3f650d53bfcc1475225d75ebc402a61b..ee32a209203ace3e7b484c5729882d4ab57f8eab 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -585,7 +585,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { } void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { - if (optr == TSDB_BINARY_OP_ADD) { + if (optr == OP_TYPE_ADD) { switch (type) { case TSDB_DATA_TYPE_TINYINT: *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2); diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt index f1ce726d859854217f723ceca196be5ee14434ae..a5773447616d86998b7581387223b73c56062fbe 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/daemon/CMakeLists.txt @@ -1,8 +1,9 @@ aux_source_directory(src DAEMON_SRC) add_executable(taosd ${DAEMON_SRC}) -target_link_libraries( + +target_include_directories( taosd - PUBLIC dnode - PUBLIC util - PUBLIC os + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +target_link_libraries(taosd dnode config util os) diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h new file mode 100644 index 0000000000000000000000000000000000000000..19ec147769a684adb7f15e27e5f55a2484894ffa --- /dev/null +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -0,0 +1,47 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DMN_INT_H_ +#define _TD_DMN_INT_H_ + +#include "config.h" +#include "dnode.h" +#include "taoserror.h" +#include "tglobal.h" +#include "ulog.h" +#include "version.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t dmnAddLogCfg(SConfig *pCfg); +int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); +int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl); + +SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); +SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg); +SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg); + +void dmnDumpCfg(SConfig *pCfg); +void dmnPrintVersion(); +void dmnGenerateGrant(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DMN_INT_H_*/ diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c deleted file mode 100644 index ce672bf7b858c3daec87b8b09e8fbda734933571..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dnode.h" -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "ulog.h" - -static struct { - bool stop; - bool dumpConfig; - bool generateGrant; - bool printAuth; - bool printVersion; - char configDir[PATH_MAX]; -} dmn = {0}; - -void dmnSigintHandle(int signum, void *info, void *ctx) { - uInfo("singal:%d is received", signum); - dmn.stop = true; -} - -void dmnSetSignalHandle() { - taosSetSignal(SIGTERM, dmnSigintHandle); - taosSetSignal(SIGHUP, dmnSigintHandle); - taosSetSignal(SIGINT, dmnSigintHandle); - taosSetSignal(SIGABRT, dmnSigintHandle); - taosSetSignal(SIGBREAK, dmnSigintHandle); -} - -int dmnParseOption(int argc, char const *argv[]) { - tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX); - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("config file path overflow"); - return -1; - } - tstrncpy(dmn.configDir, argv[i], PATH_MAX); - } else { - printf("'-c' requires a parameter, default is %s\n", configDir); - return -1; - } - } else if (strcmp(argv[i], "-C") == 0) { - dmn.dumpConfig = true; - } else if (strcmp(argv[i], "-k") == 0) { - dmn.generateGrant = true; - } else if (strcmp(argv[i], "-A") == 0) { - dmn.printAuth = true; - } else if (strcmp(argv[i], "-V") == 0) { - dmn.printVersion = true; - } else { - } - } - - return 0; -} - -void dmnGenerateGrant() { -#if 0 - grantParseParameter(); -#endif -} - -void dmnPrintVersion() { -#ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("gitinfoI: %s\n", gitinfoOfInternal); - printf("builuInfo: %s\n", buildinfo); -} - -int dmnReadConfig(const char *path) { - tstrncpy(configDir, dmn.configDir, PATH_MAX); - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - - if (taosMkDir(tsLogDir) != 0) { - printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); - return -1; - } - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosInitNotes() != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosReadCfgFromFile() != 0) { - uError("failed to read config"); - return -1; - } - - if (taosCheckAndPrintCfg() != 0) { - uError("failed to check config"); - return -1; - } - - taosSetCoreDump(tsEnableCoreFile); - return 0; -} - -void dmnDumpConfig() { taosDumpGlobalCfg(); } - -void dmnWaitSignal() { - dmnSetSignalHandle(); - while (!dmn.stop) { - taosMsleep(100); - } -} - -void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { - pCfg->sver = 30000000; // 3.0.0.0 - pCfg->numOfCores = tsNumOfCores; - pCfg->numOfCommitThreads = tsNumOfCommitThreads; - pCfg->enableTelem = 0; - tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); - tstrncpy(pCfg->buildinfo, buildinfo, 64); - tstrncpy(pCfg->gitinfo, gitinfo, 48); -} - -void dmnInitObjCfg(SDnodeObjCfg *pCfg) { - pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; - pCfg->statusInterval = tsStatusInterval; - pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; - pCfg->ratioOfQueryCores = tsRatioOfQueryCores; - pCfg->maxShellConns = tsMaxShellConns; - pCfg->shellActivityTimer = tsShellActivityTimer; - pCfg->serverPort = tsServerPort; - tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); - tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); - tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); -} - -int dmnRunDnode() { - SDnodeEnvCfg envCfg = {0}; - SDnodeObjCfg objCfg = {0}; - - dnmInitEnvCfg(&envCfg); - dmnInitObjCfg(&objCfg); - - if (dndInit(&envCfg) != 0) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - return -1; - } - - SDnode *pDnode = dndCreate(&objCfg); - if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - return -1; - } - - uInfo("Started TDengine service successfully."); - dmnWaitSignal(); - uInfo("TDengine is shut down!"); - - dndClose(pDnode); - dndCleanup(); - taosCloseLog(); - return 0; -} - -int main(int argc, char const *argv[]) { - if (dmnParseOption(argc, argv) != 0) { - return -1; - } - - if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; - } - - if (dmn.printVersion) { - dmnPrintVersion(); - return 0; - } - - if (dmnReadConfig(dmn.configDir) != 0) { - return -1; - } - - if (dmn.dumpConfig) { - dmnDumpConfig(); - return 0; - } - - return dmnRunDnode(); -} diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c new file mode 100644 index 0000000000000000000000000000000000000000..65c3e67d974940448eb9f7d40bca432e6543d79a --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +static int32_t dmnAddEpCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + if (taosGetFqdn(defaultFqdn) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; + + int32_t defaultServerPort = 6030; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1; + + return 0; +} + +static int32_t dmnAddDirCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "dataDir", osDataDir()) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", osTempDir()) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000) != 0) return -1; + return 0; +} + +static int32_t dmnCheckDirCfg(SConfig *pCfg) { + osSetDataDir(cfgGetItem(pCfg, "dataDir")->str); + osSetTempDir(cfgGetItem(pCfg, "tempDir")->str); + osSetTempReservedSpace(cfgGetItem(pCfg, "minimalDataDirGB")->fval); + osSetDataReservedSpace(cfgGetItem(pCfg, "minimalTempDirGB")->fval); + return 0; +} + +static int32_t dmnAddVersionCfg(SConfig *pCfg) { + if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; + if (cfgAddString(pCfg, "version", version) != 0) return -1; + return 0; +} + +static int32_t dmnAddDnodeCfg(SConfig *pCfg) { + if (dmnAddEpCfg(pCfg) != 0) return -1; + if (dmnAddDirCfg(pCfg) != 0) return -1; + if (dmnAddVersionCfg(pCfg) != 0) return -1; + + if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; + if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; + if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", 2, 1, 100000) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; + if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcTimer", 300, 100, 3000) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcMaxTime", 600, 100, 7200) != 0) return -1; + + return 0; +} + +static void dmnSetDnodeCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); + osSetTimezone(pItem->str); + uDebug("timezone format changed from %s to %s", pItem->str, osTimezone()); + cfgSetItem(pCfg, "timezone", osTimezone(), pItem->stype); +} + +static int32_t dmnCheckCfg(SConfig *pCfg) { + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetCoreDump(enableCore); + + dmnSetDnodeCfg(pCfg); + + if (dmnCheckDirCfg(pCfg) != 0) { + return -1; + } + + taosGetSystemInfo(); + + + if (tsNumOfCores <= 0) { + tsNumOfCores = 1; + } + + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } + + return 0; +} + +SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return NULL; + + if (dmnAddLogCfg(pCfg) != 0) { + uError("failed to add log cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (dmnAddDnodeCfg(pCfg) != 0) { + uError("failed to init dnode cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (dmnCheckCfg(pCfg) != 0) { + uError("failed to check cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + cfgDumpCfg(pCfg); + return pCfg; +} + +void dmnDumpCfg(SConfig *pCfg) { + printf("taos global config:\n"); + printf("==================================\n"); + + SConfigItem *pItem = cfgIterate(pCfg, NULL); + while (pItem != NULL) { + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + printf("cfg:%s, value:%u src:%s\n", pItem->name, pItem->bval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT32: + printf("cfg:%s, value:%d src:%s\n", pItem->name, pItem->i32, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT64: + printf("cfg:%s, value:%" PRId64 " src:%s\n", pItem->name, pItem->i64, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_FLOAT: + printf("cfg:%s, value:%f src:%s\n", pItem->name, pItem->fval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_IPSTR: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + printf("cfg:%s, value:%s src:%s\n", pItem->name, pItem->str, cfgStypeStr(pItem->stype)); + break; + } + pItem = cfgIterate(pCfg, pItem); + } +} + +SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg) { + SDnodeEnvCfg envCfg = {0}; + + const char *vstr = cfgGetItem(pCfg, "version")->str; + envCfg.sver = 30000000; + tstrncpy(envCfg.buildinfo, cfgGetItem(pCfg, "buildinfo")->str, sizeof(envCfg.buildinfo)); + tstrncpy(envCfg.gitinfo, cfgGetItem(pCfg, "gitinfo")->str, sizeof(envCfg.gitinfo)); + tstrncpy(envCfg.timezone, cfgGetItem(pCfg, "timezone")->str, sizeof(envCfg.timezone)); + tstrncpy(envCfg.locale, cfgGetItem(pCfg, "locale")->str, sizeof(envCfg.locale)); + tstrncpy(envCfg.charset, cfgGetItem(pCfg, "charset")->str, sizeof(envCfg.charset)); + envCfg.numOfCores = cfgGetItem(pCfg, "numOfCores")->i32; + envCfg.numOfCommitThreads = (uint16_t)cfgGetItem(pCfg, "numOfCommitThreads")->i32; + envCfg.enableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; + envCfg.rpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; + envCfg.rpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; + + return envCfg; +} + +SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) { + SDnodeObjCfg objCfg = {0}; + + objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + objCfg.statusInterval = cfgGetItem(pCfg, "statusInterval")->i32; + objCfg.numOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval; + objCfg.ratioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; + objCfg.maxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; + objCfg.shellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; + tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir)); + + tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); + tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); + objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn)); + snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); + return objCfg; +} \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnLog.c b/source/dnode/mgmt/daemon/src/dmnLog.c new file mode 100644 index 0000000000000000000000000000000000000000..147357c845fd675843f4f8fcb3b25c9b76f03614 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnLog.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +int32_t dmnAddLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "logDir", osLogDir()) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000) != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "dDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", 0, 0, 255) != 0) return -1; + return 0; +} + +int32_t dmnSetLogCfg(SConfig *pCfg) { + osSetLogDir(cfgGetItem(pCfg, "logDir")->str); + osSetLogReservedSpace(cfgGetItem(pCfg, "minimalLogDirGB")->fval); + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; + mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; + tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; + tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; + fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + + int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32; + taosSetAllDebugFlag(debugFlag); + + return 0; +} + +int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; + + if (dmnAddLogCfg(pCfg) != 0) { + printf("failed to add log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (dmnSetLogCfg(pCfg) != 0) { + printf("failed to set log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + cfgCleanup(pCfg); + return 0; +} + +int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char configDir[PATH_MAX] = {0}; + char configFile[PATH_MAX + 100] = {0}; + + taosExpandDir(inputCfgDir, configDir, PATH_MAX); + snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir); + + if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) { + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) { + uError("failed to load from config file:%s since %s\n", configFile, terrstr()); + return -1; + } + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; + } + + return 0; +} diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c new file mode 100644 index 0000000000000000000000000000000000000000..62f0db5fc52f982870153c8708ae15ef5f3cfb32 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnMain.c @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +static struct { + bool stop; + bool dumpConfig; + bool generateGrant; + bool printAuth; + bool printVersion; + char envFile[PATH_MAX]; + char apolloUrl[PATH_MAX]; +} dmn = {0}; + +static void dmnSigintHandle(int signum, void *info, void *ctx) { + uInfo("singal:%d is received", signum); + dmn.stop = true; +} + +static void dmnSetSignalHandle() { + taosSetSignal(SIGTERM, dmnSigintHandle); + taosSetSignal(SIGHUP, dmnSigintHandle); + taosSetSignal(SIGINT, dmnSigintHandle); + taosSetSignal(SIGABRT, dmnSigintHandle); + taosSetSignal(SIGBREAK, dmnSigintHandle); +} + +static void dmnWaitSignal() { + dmnSetSignalHandle(); + while (!dmn.stop) { + taosMsleep(100); + } +} + +static int32_t dmnParseOption(int32_t argc, char const *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-C") == 0) { + dmn.dumpConfig = true; + } else if (strcmp(argv[i], "-k") == 0) { + dmn.generateGrant = true; + } else if (strcmp(argv[i], "-V") == 0) { + dmn.printVersion = true; + } else { + } + } + + return 0; +} + +int32_t dmnRunDnode(SConfig *pCfg) { + SDnodeEnvCfg envCfg = dmnGetEnvCfg(pCfg); + if (dndInit(&envCfg) != 0) { + uInfo("Failed to start TDengine, please check the log"); + return -1; + } + + SDnodeObjCfg objCfg = dmnGetObjCfg(pCfg); + SDnode *pDnode = dndCreate(&objCfg); + if (pDnode == NULL) { + uInfo("Failed to start TDengine, please check the log"); + return -1; + } + + uInfo("Started TDengine service successfully."); + dmnWaitSignal(); + uInfo("TDengine is shut down!"); + + dndClose(pDnode); + dndCleanup(); + taosCloseLog(); + return 0; +} + +int main(int argc, char const *argv[]) { + osInit(); + + if (dmnParseOption(argc, argv) != 0) { + return -1; + } + + if (dmn.generateGrant) { + dmnGenerateGrant(); + return 0; + } + + if (dmn.printVersion) { + dmnPrintVersion(); + return 0; + } + + if (dmnInitLog(configDir, dmn.envFile, dmn.apolloUrl) != 0) { + return -1; + } + + SConfig *pCfg = dmnReadCfg(configDir, dmn.envFile, dmn.apolloUrl); + if (pCfg == NULL) { + uInfo("Failed to start TDengine since read config error"); + return -1; + } + + if (dmn.dumpConfig) { + dmnDumpCfg(pCfg); + cfgCleanup(pCfg); + return 0; + } + + int32_t code = dmnRunDnode(pCfg); + cfgCleanup(pCfg); + return code; +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/daemon/src/dmnUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..608bc7f3e6d0167d28cfa373983b3ceaed4041d1 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnUtil.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +void dmnGenerateGrant() { +#if 0 + grantParseParameter(); +#endif +} + +void dmnPrintVersion() { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("gitinfoI: %s\n", gitinfoOfInternal); + printf("builuInfo: %s\n", buildinfo); +} diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 02dced53c2ecf62e24f954eeb6555fb60474fd02..b8f249f838203b57232c315a3503729fe2b13449 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -270,7 +270,8 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { taosBlockSIGPIPE(); taosResolveCRC(); - if (rpcInit() != 0) { + SRpcCfg rpcCfg = {.rpcTimer = pCfg->rpcTimer, .rpcMaxTime = pCfg->rpcMaxTime, .sver = pCfg->sver}; + if (rpcInit(&rpcCfg) != 0) { dError("failed to init rpc since %s", terrstr()); dndCleanup(); return -1; @@ -325,18 +326,6 @@ void taosGetDisk() { SDiskSize diskSize = tfsGetSize(pTfs); tfsUpdateSize(&fsMeta); - tsTotalDataDirGB = (float)(fsMeta.total / unit); - tsUsedDataDirGB = (float)(fsMeta.used / unit); - tsAvailDataDirGB = (float)(fsMeta.avail / unit); - if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { - tsTotalLogDirGB = (float)(diskSize.total / unit); - tsAvailLogDirGB = (float)(diskSize.avail / unit); - } - - if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { - tsTotalTmpDirGB = (float)(diskSize.total / unit); - tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); - } #endif } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 3d149a3e60ce49c6334ac76d42371d001227a7b4..b7f42fb465623687f468bb51874d1520f17abc7b 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -296,7 +296,7 @@ PRASE_DNODE_OVER: if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep); + taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pDnode->cfg.serverPort, &dnodeEp.ep); taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); } @@ -371,9 +371,9 @@ void dndSendStatusReq(SDnode *pDnode) { req.clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN); - memcpy(req.clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN); + memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, pDnode->env.locale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, pDnode->env.charset, TD_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 4e35baf9052bdd11fffee289e59f00255bf96634..0aae145d2f3c9ab67c3546bb7cfa3dab49a141b6 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -25,8 +25,8 @@ #include "dndMnode.h" #include "dndVnodes.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dndInitMsgFp(STransMgmt *pMgmt) { @@ -113,6 +113,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; @@ -155,7 +156,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode * pDnode = parent; + SDnode *pDnode = parent; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pRsp->msgType; @@ -219,7 +220,7 @@ static void dndCleanupClient(SDnode *pDnode) { } static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode * pDnode = param; + SDnode *pDnode = param; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pReq->msgType; @@ -313,7 +314,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 250d563a8b3a4d617a5e2ac37a3c6fbd063ba99f..c5c7ff2920d32f090d880afe41103e0178e0d341 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -20,11 +20,10 @@ #include "os.h" #include "dnode.h" -#include "tconfig.h" +#include "tmsg.h" #include "tdataformat.h" #include "tglobal.h" #include "tmsg.h" -#include "tnote.h" #include "trpc.h" #include "tthread.h" #include "ulog.h" diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 771c5886efe179f26a6355a10888fc8244ab235b..5d2abd86c35cbe3a2ef8e227c131dc58b416782d 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -28,16 +28,13 @@ void Testbase::InitLog(const char* path) { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; taosRemoveDir(path); taosMkDir(path); - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + osSetLogDir(path); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } } @@ -46,6 +43,8 @@ void Testbase::Init(const char* path, int16_t port) { SDnodeEnvCfg cfg = {0}; cfg.numOfCommitThreads = 1; cfg.numOfCores = 1; + cfg.rpcMaxTime = 600; + cfg.rpcTimer = 300; dndInit(&cfg); char fqdn[] = "localhost"; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e64191f7153c87301fa14d74e44cdb75b5f2cd89..bb2367ef728398ae049448afb42f544dccf71c90 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -123,6 +123,7 @@ typedef enum { TRN_TYPE_DROP_TOPIC = 1015, TRN_TYPE_SUBSCRIBE = 1016, TRN_TYPE_REBALANCE = 1017, + TRN_TYPE_COMMIT_OFFSET = 1018, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -176,7 +177,7 @@ typedef struct { SArray* undoActions; int64_t createdTime; int64_t lastExecTime; - uint64_t dbUid; + int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; } STrans; @@ -304,16 +305,16 @@ typedef struct { } SDbCfg; typedef struct { - char name[TSDB_DB_FNAME_LEN]; - char acct[TSDB_USER_LEN]; - char createUser[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - uint64_t uid; - int32_t cfgVersion; - int32_t vgVersion; - int8_t hashMethod; // default is 1 - SDbCfg cfg; + char name[TSDB_DB_FNAME_LEN]; + char acct[TSDB_USER_LEN]; + char createUser[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int64_t uid; + int32_t cfgVersion; + int32_t vgVersion; + int8_t hashMethod; // default is 1 + SDbCfg cfg; } SDbObj; typedef struct { @@ -346,8 +347,8 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t createdTime; int64_t updateTime; - uint64_t uid; - uint64_t dbUid; + int64_t uid; + int64_t dbUid; int32_t version; int32_t nextColId; int32_t numOfColumns; @@ -465,6 +466,24 @@ static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) { } } +typedef struct { + char key[TSDB_PARTITION_KEY_LEN]; + int64_t offset; +} SMqOffsetObj; + +static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pOffset->key); + tlen += taosEncodeFixedI64(buf, pOffset->offset); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) { + buf = taosDecodeStringTo(buf, pOffset->key); + buf = taosDecodeFixedI64(buf, &pOffset->offset); + return buf; +} + typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; int32_t status; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 97cd6caf0476e4a275383d463caaf9d392416746..5c32da966b5fb84e96d351d1174c231656c301d6 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -56,12 +56,9 @@ typedef struct { } SProfileMgmt; typedef struct { - int8_t enable; - pthread_mutex_t lock; - pthread_cond_t cond; - volatile int32_t exit; - pthread_t thread; - char email[TSDB_FQDN_LEN]; + bool enable; + SRWLatch lock; + char email[TSDB_FQDN_LEN]; } STelemMgmt; typedef struct { @@ -81,6 +78,7 @@ typedef struct SMnode { tmr_h timer; tmr_h transTimer; tmr_h mqTimer; + tmr_h telemTimer; char *path; SMnodeCfg cfg; int64_t checkTime; diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h new file mode 100644 index 0000000000000000000000000000000000000000..e18eec973f87129570983ad4724df115c6e18276 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_OFFSET_H_ +#define _TD_MND_OFFSET_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitOffset(SMnode *pMnode); +void mndCleanupOffset(SMnode *pMnode); + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key); +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset); + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset); +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw); + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); + +static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { + return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); +} + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_OFFSET_H_*/ diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index a41ccd989690847c3e9e20632f372b0b5a6583a2..3410a386da2d80bafcb35db5f2d97e987f753eb3 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -52,7 +52,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { SSdb *pSdb = pMnode->pSdb; SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId); - if (pCluster = NULL) { + if (pCluster == NULL) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c new file mode 100644 index 0000000000000000000000000000000000000000..ac9e99ebd43769559c47868275d84da2c4066001 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndOffset.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define MND_OFFSET_VER_NUMBER 1 +#define MND_OFFSET_RESERVE_SIZE 64 + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset); +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq); + +int32_t mndInitOffset(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupOffset(SMnode *pMnode) {} + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset); + int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size); + if (pRaw == NULL) goto OFFSET_ENCODE_OVER; + + buf = malloc(tlen); + if (buf == NULL) goto OFFSET_ENCODE_OVER; + + void *abuf = buf; + tEncodeSMqOffsetObj(&abuf, pOffset); + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER); + SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER); + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_ENCODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRaw; +} + +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER; + + if (sver != MND_OFFSET_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto OFFSET_DECODE_OVER; + } + + int32_t size = sizeof(SMqOffsetObj); + SSdbRow *pRow = sdbAllocRow(size); + if (pRow == NULL) goto OFFSET_DECODE_OVER; + + SMqOffsetObj *pOffset = sdbGetRowObj(pRow); + if (pOffset == NULL) goto OFFSET_DECODE_OVER; + + int32_t dataPos = 0; + int32_t tlen; + SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER); + buf = malloc(tlen + 1); + if (buf == NULL) goto OFFSET_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER); + SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER); + + if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) { + goto OFFSET_DECODE_OVER; + } + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_DECODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr()); + tfree(pRow); + return NULL; + } + + mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRow; +} + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { + int32_t code = 0; + int32_t sz = taosArrayGetSize(vgs); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i); + SMqOffsetObj offsetObj; + if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) { + return -1; + } + offsetObj.offset = -1; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj); + if (pOffsetRaw == NULL) { + return -1; + } + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) { + return -1; + } + } + return 0; +} + +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) { + char key[TSDB_PARTITION_KEY_LEN]; + + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SMqCMCommitOffsetReq commitOffsetReq; + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, msgStr, pMsg->rpcMsg.contLen, TD_DECODER); + + tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, &pMsg->rpcMsg); + + for (int32_t i = 0; i < commitOffsetReq.num; i++) { + SMqOffset *pOffset = &commitOffsetReq.offsets[i]; + if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { + return -1; + } + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key); + ASSERT(pOffsetObj); + pOffsetObj->offset = pOffset->offset; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj); + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + mndTransAppendRedolog(pTrans, pOffsetRaw); + mndReleaseOffset(pMnode, pOffsetObj); + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform insert action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform delete action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) { + mTrace("offset:%s, perform update action", pOldOffset->key); + return 0; +} + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) { + SSdb *pSdb = pMnode->pSdb; + SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key); + if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST; + } + return pOffset; +} + +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pOffset); +} + +static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index df892e224237e7eb5bf4d2c538397620c2259c80..2dfdc8dc7ffb03c4ed9a55a7ee0da6ce42d967f1 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -21,6 +21,7 @@ #include "mndStb.h" #include "mndUser.h" #include "tglobal.h" +#include "version.h" #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a5c4c41244e32d79d99b6072601267f4e923e4bb..2ea157fea4407259ba86a1060c7a3dc409e45b93 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -19,6 +19,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndStb.h" #include "mndTopic.h" @@ -80,13 +81,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *consumerGroup) { +static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) { SMqSubscribeObj *pSub = tNewSubscribeObj(); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - char *key = mndMakeSubscribeKey(consumerGroup, pTopic->name); + char *key = mndMakeSubscribeKey(cgroup, pTopic->name); if (key == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSMqSubscribeObj(pSub); @@ -289,9 +290,15 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); for (int32_t k = 0; k < vgsz; k++) { + char offsetKey[TSDB_PARTITION_KEY_LEN]; SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - - SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId}; + SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1}; + mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); + if (pOffsetObj != NULL) { + vgEp.offset = pOffsetObj->offset; + mndReleaseOffset(pMnode, pOffsetObj); + } taosArrayPush(topicEp.vgs, &vgEp); } taosArrayPush(rsp.topics, &topicEp); @@ -870,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SUB_ENCODE_OVER: tfree(buf); - if (terrno != 0) { + if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); return NULL; @@ -1085,6 +1092,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName); pSub = mndCreateSubscription(pMnode, pTopic, cgroup); createSub = true; + + mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg); } SMqSubConsumer mqSubConsumer; diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 5beb1b10e36bc954b9283113b91cf7ca4ab4aa01..5292dd0a416437beae01554fa7f8648fb3989940 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -21,8 +21,8 @@ #include "tversion.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" -#define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 +#define TELEMETRY_PORT 80 +#define REPORT_INTERVAL 86400 static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } @@ -33,24 +33,7 @@ static void mndCloseObject(SBufferWriter* bw) { } else { tbufWriteChar(bw, '}'); } - tbufWriteChar(bw, ','); -} - -#if 0 -static void beginArray(SBufferWriter* bw) { - tbufWriteChar(bw, '['); -} - -static void closeArray(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, ']'); - } else { - tbufWriteChar(bw, ']'); - } - tbufWriteChar(bw, ','); } -#endif static void mndWriteString(SBufferWriter* bw, const char* str) { tbufWriteChar(bw, '"'); @@ -61,7 +44,7 @@ static void mndWriteString(SBufferWriter* bw, const char* str) { static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) { mndWriteString(bw, k); tbufWriteChar(bw, ':'); - char buf[32]; + char buf[32] = {0}; sprintf(buf, "%" PRId64, v); tbufWrite(bw, buf, strlen(buf)); tbufWriteChar(bw, ','); @@ -184,24 +167,17 @@ static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) { } static void mndSendTelemetryReport(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - - char buf[128] = {0}; - uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); - if (ip == 0xffffffff) { - mDebug("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); - return; - } - SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); - if (fd < 0) { - mDebug("failed to create socket for telemetry, reason:%s", strerror(errno)); - return; - } + STelemMgmt* pMgmt = &pMnode->telemMgmt; + SBufferWriter bw = tbufInitWriter(NULL, false); + int32_t code = -1; + char buf[128] = {0}; + SOCKET fd = 0; char clusterName[64] = {0}; - mndGetClusterName(pMnode, clusterName, sizeof(clusterName)); + if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) { + goto SEND_OVER; + } - SBufferWriter bw = tbufInitWriter(NULL, false); mndBeginObject(&bw); mndAddStringField(&bw, "instanceId", clusterName); mndAddIntField(&bw, "reportVersion", 1); @@ -212,54 +188,69 @@ static void mndSendTelemetryReport(SMnode* pMnode) { mndAddRuntimeInfo(pMnode, &bw); mndCloseObject(&bw); + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); + if (ip == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr()); + goto SEND_OVER; + } + + fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + goto SEND_OVER; + } + const char* header = "POST /report HTTP/1.1\n" "Host: " TELEMETRY_SERVER "\n" "Content-Type: application/json\n" "Content-Length: "; + if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) { + goto SEND_OVER; + } - taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)); - int32_t contLen = (int32_t)(tbufTell(&bw) - 1); + int32_t contLen = (int32_t)(tbufTell(&bw)); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, (int32_t)strlen(buf)); - taosWriteSocket(fd, tbufGetData(&bw, false), contLen); - tbufCloseWriter(&bw); + if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) { + goto SEND_OVER; + } + + const char* pCont = tbufGetData(&bw, false); + if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + goto SEND_OVER; + } // read something to avoid nginx error 499 if (taosReadSocket(fd, buf, 10) < 0) { - mDebug("failed to receive response since %s", strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to receive response since %s", terrstr()); + goto SEND_OVER; } + mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont); + code = 0; + +SEND_OVER: + tbufCloseWriter(&bw); taosCloseSocket(fd); + + if (code != 0) { + mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + } } -static void* mndTelemThreadFp(void* param) { - SMnode* pMnode = param; +static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { + SMnode* pMnode = pReq->pMnode; STelemMgmt* pMgmt = &pMnode->telemMgmt; + if (!pMgmt->enable) return 0; - struct timespec end = {0}; - clock_gettime(CLOCK_REALTIME, &end); - end.tv_sec += 300; // wait 5 minutes before send first report - - setThreadName("mnd-telem"); - - while (!pMgmt->exit) { - int32_t r = 0; - struct timespec ts = end; - pthread_mutex_lock(&pMgmt->lock); - r = pthread_cond_timedwait(&pMgmt->cond, &pMgmt->lock, &ts); - pthread_mutex_unlock(&pMgmt->lock); - if (r == 0) break; - if (r != ETIMEDOUT) continue; - - if (mndIsMaster(pMnode)) { - mndSendTelemetryReport(pMnode); - } - end.tv_sec += REPORT_INTERVAL; - } - - return NULL; + taosWLockLatch(&pMgmt->lock); + mndSendTelemetryReport(pMnode); + taosWUnLockLatch(&pMgmt->lock); + return 0; } static void mndGetEmail(SMnode* pMnode, char* filepath) { @@ -280,43 +271,12 @@ static void mndGetEmail(SMnode* pMnode, char* filepath) { int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; pMgmt->enable = pMnode->cfg.enableTelem; - - if (!pMgmt->enable) return 0; - - pMgmt->exit = 0; - pthread_mutex_init(&pMgmt->lock, NULL); - pthread_cond_init(&pMgmt->cond, NULL); - pMgmt->email[0] = 0; - + taosInitRWLatch(&pMgmt->lock); mndGetEmail(pMnode, "/usr/local/taos/email"); - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode); - pthread_attr_destroy(&attr); - if (code != 0) { - mDebug("failed to create telemetry thread since :%s", strerror(code)); - } - - mInfo("mnd telemetry is initialized"); + mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); + mDebug("mnode telemetry is initialized"); return 0; } -void mndCleanupTelem(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - if (!pMgmt->enable) return; - - if (taosCheckPthreadValid(pMgmt->thread)) { - pthread_mutex_lock(&pMgmt->lock); - pMgmt->exit = 1; - pthread_cond_signal(&pMgmt->cond); - pthread_mutex_unlock(&pMgmt->lock); - - pthread_join(pMgmt->thread, NULL); - } - - pthread_mutex_destroy(&pMgmt->lock); - pthread_cond_destroy(&pMgmt->cond); -} +void mndCleanupTelem(SMnode* pMnode) {} diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7d684cd5427de374dec1693f347260fc3dbc7672..a009e01a52cfd1a535529cd71c47235b000a828a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -71,7 +71,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndTransActionUpdate, .deleteFp = (SdbDeleteFp)mndTransActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndGetTransMeta); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 64b4aa6dd702d6477a710b42bf3e9c82e1937d8c..299e66a7c011723bc3c43940d4041e0fc98bc0e9 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -23,6 +23,7 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" @@ -36,6 +37,10 @@ #include "mndUser.h" #include "mndVgroup.h" +#define MQ_TIMER_MS 3000 +#define TRNAS_TIMER_MS 6000 +#define TELEM_TIMER_MS 86400000 + int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { terrno = TSDB_CODE_MND_NOT_READY; @@ -73,46 +78,60 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } -static void mndTransReExecute(void *param, void *tmrId) { +static void mndPullupTrans(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen}; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); +} + +static void mndPullupTelem(void *param, void *tmrId) { + SMnode *pMnode = param; + if (mndIsMaster(pMnode)) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; + pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + } + + taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); } static int32_t mndInitTimer(SMnode *pMnode) { + pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); if (pMnode->timer == NULL) { - pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - if (pMnode->timer == NULL) { + if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndTransReExecute, 6000, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -126,6 +145,8 @@ static void mndCleanupTimer(SMnode *pMnode) { pMnode->transTimer = NULL; taosTmrStop(pMnode->mqTimer); pMnode->mqTimer = NULL; + taosTmrStop(pMnode->telemTimer); + pMnode->telemTimer = NULL; taosTmrCleanUp(pMnode->timer); pMnode->timer = NULL; } @@ -197,6 +218,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; @@ -404,8 +426,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return NULL; } - if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && - pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { + if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && + pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) { SRpcConnInfo connInfo = {0}; if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { taosFreeQitem(pMsg); @@ -440,7 +462,7 @@ void mndProcessMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; int32_t code = 0; tmsg_t msgType = pMsg->rpcMsg.msgType; - void * ahandle = pMsg->rpcMsg.ahandle; + void *ahandle = pMsg->rpcMsg.ahandle; bool isReq = (msgType & 1U); mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4dc46b37984ba4b5976ec45130fa7725b001a369..ac9dde35974227538b4c8aee7ba5d15ecfeee9bb 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -207,9 +207,17 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; - int64_t fetchOffset = pReq->offset; + int64_t fetchOffset; /*int64_t blockingTime = pReq->blockingTime;*/ + if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { + fetchOffset = 0; + } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { + fetchOffset = walGetLastVer(pTq->pWal); + } else { + fetchOffset = pReq->currentOffset + 1; + } + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); @@ -226,31 +234,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0); ASSERT(pConsumer->consumerId == consumerId); - if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { - pTopic->committedOffset = pReq->offset; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = 0; - rpcSendResponse(pMsg); - return 0; - } - - if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { - if (pTopic->committedOffset < pReq->offset - 1) { - pTopic->committedOffset = pReq->offset - 1; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - } - } - - rsp.committedOffset = pTopic->committedOffset; - rsp.reqOffset = pReq->offset; + rsp.reqOffset = pReq->currentOffset; rsp.skipLogNum = 0; - if (fetchOffset <= pTopic->committedOffset) { - fetchOffset = pTopic->committedOffset + 1; - } - SWalHead* pHead; while (1) { int8_t pos = fetchOffset % TQ_BUFFER_SIZE; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ade280eecc85f9346331bf5a91a79c8afbdf1198..c3947da4591de26b9a4efbc6cb5b23204e8fbb49 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -22,14 +22,15 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); - // ser request version + // set request version void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedI64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { - /*ASSERT(false);*/ // TODO: handle error + /*ASSERT(false);*/ + vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr()); } } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index a2454e4c5e731c872061eb97a1a076352b7c8a0c..da2ab5b4250a1e1be2f13fff4ef989ddca9c3037 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -14,4 +14,5 @@ add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) add_subdirectory(nodes) -add_subdirectory(scalar) \ No newline at end of file +add_subdirectory(config) +add_subdirectory(scalar) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d4da79f9681a7a00540700263ff463603033484b..d62c189d33cd7c7621f8c65d3cf8981b9cf754b2 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -546,7 +546,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); + ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 1c84f06a2243b56f7e3427864bd3fc502dcc5343..9dfe6eac3fcc06e8210191513057e35398cd1526 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -131,10 +131,8 @@ void ctgTestInitLogFile() { ctgDbgEnableDebug("api"); - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", osLogDir()); } } diff --git a/source/libs/config/CMakeLists.txt b/source/libs/config/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..596cc0b433fdca4c509ee569845e3c4cea08873a --- /dev/null +++ b/source/libs/config/CMakeLists.txt @@ -0,0 +1,13 @@ +aux_source_directory(src CONFIG_SRC) +add_library(config STATIC ${CONFIG_SRC}) +target_include_directories( + config + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/config" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(config os util common) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h new file mode 100644 index 0000000000000000000000000000000000000000..6497393792210f0db7a5bcbacdadafee0cfb0e1c --- /dev/null +++ b/source/libs/config/inc/cfgInt.h @@ -0,0 +1,47 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_CFG_INT_H_ +#define _TD_CFG_INT_H_ + +#include "config.h" +#include "taoserror.h" +#include "thash.h" +#include "tutil.h" +#include "ulog.h" +#include "tglobal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SConfig { + ECfgSrcType stype; + SHashObj *hash; +} SConfig; + +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvVar(SConfig *pConfig); +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); + +int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_CFG_INT_H_*/ diff --git a/include/common/tlocale.h b/source/libs/config/src/cfgApolloUrl.c similarity index 77% rename from include/common/tlocale.h rename to source/libs/config/src/cfgApolloUrl.c index d809c75d67983fb6ac844a656a361876ea7d3504..f35eca70c385407f7dc7d3504089ac825faf6721 100644 --- a/include/common/tlocale.h +++ b/source/libs/config/src/cfgApolloUrl.c @@ -13,17 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_LOCALE_H_ -#define _TD_COMMON_LOCALE_H_ +#define _DEFAULT_SOURCE +#include "cfgInt.h" -#ifdef __cplusplus -extern "C" { -#endif - -void tsSetLocale(); - -#ifdef __cplusplus -} -#endif /*_TD_COMMON_LOCALE_H_*/ - -#endif +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { + uInfo("load from apoll url %s", url); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/cfgCfgFile.c b/source/libs/config/src/cfgCfgFile.c new file mode 100644 index 0000000000000000000000000000000000000000..4eb835be7ff9362c82c8a7c99d24ce37a91f7d32 --- /dev/null +++ b/source/libs/config/src/cfgCfgFile.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { + char *line, *name, *value, *value2, *value3; + int olen, vlen, vlen2, vlen3; + ssize_t _bytes = 0; + size_t len = 1024; + + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + line = malloc(len); + + while (!feof(fp)) { + memset(line, 0, len); + + name = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; + + _bytes = tgetline(&line, &len, fp); + if (_bytes < 0) { + break; + } + + line[len - 1] = 0; + + paGetToken(line, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; + + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + + cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + // taosReadConfigOption(name, value, value2, value3); + } + + fclose(fp); + tfree(line); + + uInfo("load from cfg file %s success", filepath); + return 0; +} \ No newline at end of file diff --git a/include/common/ttimezone.h b/source/libs/config/src/cfgEnvFile.c similarity index 77% rename from include/common/ttimezone.h rename to source/libs/config/src/cfgEnvFile.c index cc50a29d57a2f1a308b8b236fc7bb80122adb387..a12d1cd1d5aa93ab4b439a730be7306b61b33cf0 100644 --- a/include/common/ttimezone.h +++ b/source/libs/config/src/cfgEnvFile.c @@ -13,17 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_TIMEZONE_H_ -#define _TD_COMMON_TIMEZONE_H_ +#define _DEFAULT_SOURCE +#include "cfgInt.h" -#ifdef __cplusplus -extern "C" { -#endif - -void tsSetTimeZone(); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_COMMON_TIMEZONE_H_*/ +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { + uInfo("load from env file %s", filepath); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/cfgEnvVar.c b/source/libs/config/src/cfgEnvVar.c new file mode 100644 index 0000000000000000000000000000000000000000..c3b3df404742ec35bf51f4a72ec314d731bc438d --- /dev/null +++ b/source/libs/config/src/cfgEnvVar.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromEnvVar(SConfig *pConfig) { + uInfo("load from global env variables"); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c new file mode 100644 index 0000000000000000000000000000000000000000..b23826ceacc696669af94c2dbc729897e2fa5efd --- /dev/null +++ b/source/libs/config/src/config.c @@ -0,0 +1,633 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" +#include "tep.h" +#include "tmsg.h" + +#define CFG_NAME_PRINT_LEN 22 +#define CFG_SRC_PRINT_LEN 12 + +SConfig *cfgInit() { + SConfig *pCfg = calloc(1, sizeof(SConfig)); + if (pCfg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pCfg->hash = taosHashInit(16, MurmurHash3_32, false, HASH_NO_LOCK); + if (pCfg->hash == NULL) { + free(pCfg); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + return pCfg; +} + +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { + switch (cfgType) { + case CFG_STYPE_CFG_FILE: + return cfgLoadFromCfgFile(pCfg, sourceStr); + case CFG_STYPE_ENV_FILE: + return cfgLoadFromEnvFile(pCfg, sourceStr); + case CFG_STYPE_ENV_VAR: + return cfgLoadFromEnvVar(pCfg); + case CFG_STYPE_APOLLO_URL: + return cfgLoadFromApollUrl(pCfg, sourceStr); + default: + return -1; + } +} + +void cfgCleanup(SConfig *pCfg) { + if (pCfg != NULL) { + if (pCfg->hash != NULL) { + taosHashCleanup(pCfg->hash); + pCfg->hash == NULL; + } + free(pCfg); + } +} + +int32_t cfgGetSize(SConfig *pCfg) { return taosHashGetSize(pCfg->hash); } + +SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashIterate(pCfg->hash, pIter); } + +void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashCancelIterate(pCfg->hash, pIter); } + +static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { + tfree(pItem->str); + pItem->str = strdup(timezone); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { + tfree(pItem->str); + pItem->str = strdup(charset); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) { + tfree(pItem->str); + pItem->str = strdup(locale); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { + char fullDir[PATH_MAX] = {0}; + if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to expand dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosRealPath(fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosMkDir(fullDir) != 0) { + uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); + return -1; + } + + tfree(pItem->str); + pItem->str = strdup(fullDir); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetIpStr(SConfigItem *pItem, const char *ip) { + uint32_t value = taosInetAddr(ip); + if (value == INADDR_NONE) { + uError("ip:%s is not a valid ip address", ip); + return -1; + } + + tfree(pItem->str); + pItem->str = strdup(ip); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + bool tmp = false; + if (strcasecmp(value, "true") == 0) { + tmp = true; + } + if (atoi(value) > 0) { + tmp = true; + } + + pItem->bval = tmp; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int32_t ival = (int32_t)atoi(value); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i32); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->i32 = ival; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int64_t ival = (int64_t)atoi(value); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 + "], use last src:%s value:%" PRId64, + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i64); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->i64 = ival; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + float fval = (float)atof(value); + if (fval < pItem->fmin || fval > pItem->fmax) { + uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->fmin, pItem->fmax, cfgStypeStr(pItem->stype), + pItem->fval); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->fval = fval; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + free(pItem->str); + pItem->str = tmp; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetIpStr(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetIpStr(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetDir(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetLocale(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetCharset(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetTimezone(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) { + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (pItem == NULL) { + return -1; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + return cfgSetBool(pItem, value, stype); + case CFG_DTYPE_INT32: + return cfgSetInt32(pItem, value, stype); + case CFG_DTYPE_INT64: + return cfgSetInt64(pItem, value, stype); + case CFG_DTYPE_FLOAT: + return cfgSetFloat(pItem, value, stype); + case CFG_DTYPE_STRING: + return cfgSetString(pItem, value, stype); + case CFG_DTYPE_IPSTR: + return cfgSetIpStr(pItem, value, stype); + case CFG_DTYPE_DIR: + return cfgSetDir(pItem, value, stype); + case CFG_DTYPE_TIMEZONE: + return cfgSetTimezone(pItem, value, stype); + case CFG_DTYPE_CHARSET: + return cfgSetCharset(pItem, value, stype); + case CFG_DTYPE_LOCALE: + return cfgSetLocale(pItem, value, stype); + case CFG_DTYPE_NONE: + default: + break; + } + + terrno = TSDB_CODE_INVALID_CFG; + return -1; +} + +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); + strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); + + SConfigItem *pItem = taosHashGet(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1); + if (pItem == NULL) { + terrno = TSDB_CODE_CFG_NOT_FOUND; + } + + return pItem; +} + +static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { + pItem->stype = CFG_STYPE_DEFAULT; + pItem->name = strdup(name); + if (pItem->name == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); + strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); + + if (taosHashPut(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { + if (pItem->dtype == CFG_DTYPE_STRING) { + free(pItem->str); + } + free(pItem->name); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING}; + item.str = strdup(defaultVal); + if (item.str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; + if (cfgCheckAndSetIpStr(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR}; + if (cfgCheckAndSetDir(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE}; + if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET}; + if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE}; + if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +const char *cfgStypeStr(ECfgSrcType type) { + switch (type) { + case CFG_STYPE_DEFAULT: + return "default"; + case CFG_STYPE_CFG_FILE: + return "cfg_file"; + case CFG_STYPE_ENV_FILE: + return "env_file"; + case CFG_STYPE_ENV_VAR: + return "env_var"; + case CFG_STYPE_APOLLO_URL: + return "apollo_url"; + case CFG_STYPE_ARG_LIST: + return "arg_list"; + case CFG_STYPE_API_OPTION: + return "api_option"; + default: + return "invalid"; + } +} + +const char *cfgDtypeStr(ECfgDataType type) { + switch (type) { + case CFG_DTYPE_NONE: + return "none"; + case CFG_DTYPE_BOOL: + return "bool"; + case CFG_DTYPE_INT32: + return "int32"; + case CFG_DTYPE_INT64: + return "int64"; + case CFG_DTYPE_FLOAT: + return "float"; + case CFG_DTYPE_STRING: + return "string"; + case CFG_DTYPE_IPSTR: + return "ipstr"; + case CFG_DTYPE_DIR: + return "dir"; + case CFG_DTYPE_LOCALE: + return "locale"; + case CFG_DTYPE_CHARSET: + return "charset"; + case CFG_DTYPE_TIMEZONE: + return "timezone"; + default: + return "invalid"; + } +} + +void cfgDumpCfg(SConfig *pCfg) { + uInfo(" global config"); + uInfo("================================================================="); + + char src[CFG_SRC_PRINT_LEN + 1] = {0}; + char name[CFG_NAME_PRINT_LEN + 1] = {0}; + + SConfigItem *pItem = cfgIterate(pCfg, NULL); + while (pItem != NULL) { + tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); + for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { + if (src[i] == 0) src[i] = ' '; + } + + tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN); + for (int32_t i = 0; i < CFG_NAME_PRINT_LEN; ++i) { + if (name[i] == 0) name[i] = ' '; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + uInfo("%s %s %u", src, name, pItem->bval); + break; + case CFG_DTYPE_INT32: + uInfo("%s %s %d", src, name, pItem->i32); + break; + case CFG_DTYPE_INT64: + uInfo("%s %s %" PRId64, src, name, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + uInfo("%s %s %f", src, name, pItem->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_IPSTR: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + uInfo("%s %s %s", src, name, pItem->str); + break; + } + pItem = cfgIterate(pCfg, pItem); + } + + uInfo("================================================================="); +} +#if 0 +// int32_t cfgCheck(SConfig *pCfg) { +// SConfigItem *pItem = NULL; + +// pItem = cfgGetItem(pCfg, "serverPort"); +// if (pItem != NULL) { +// tsServerPort = (uint16_t)pItem->i32; +// } + +// pItem = cfgGetItem(pCfg, "firstEp"); +// if (pItem != NULL) { +// tstrncpy(tsFirst, pItem->str, TSDB_EP_LEN); +// } + +// snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); +// uInfo("localEp is: %s", tsLocalEp); + +// SEp ep = {0}; +// if (tsFirst[0] == 0) { +// strcpy(tsFirst, tsLocalEp); +// } else { +// taosGetFqdnPortFromEp(tsFirst, &ep); +// snprintf(tsFirst, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); +// } + +// pItem = cfgGetItem(pCfg, "secondEp"); +// if (pItem != NULL) { +// tstrncpy(tsSecond, pItem->str, TSDB_EP_LEN); +// } + +// if (tsSecond[0] == 0) { +// strcpy(tsSecond, tsLocalEp); +// } else { +// taosGetFqdnPortFromEp(tsSecond, &ep); +// snprintf(tsSecond, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); +// } + +// pItem = cfgGetItem(pCfg, "dataDir"); +// if (pItem != NULL) { +// tstrncpy(osDataDir(), pItem->str, PATH_MAX); +// } + +// if (tsDiskCfgNum <= 0) { +// taosAddDataDir(0, osDataDir(), 0, 1); +// tsDiskCfgNum = 1; +// uTrace("dataDir:%s, level:0 primary:1 is configured by default", osDataDir()); +// } + +// if (taosDirExist(osTempDir()) != 0) { +// return -1; +// } + +// taosGetSystemInfo(); + +// tsSetLocale(); + +// // SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); +// // if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { +// tsSetTimeZone(); +// // } + +// pItem = cfgGetItem(pCfg, "numOfCores"); +// if (pItem != NULL) { +// tsNumOfCores = pItem->i32; +// } + +// if (tsNumOfCores <= 0) { +// tsNumOfCores = 1; +// } + +// if (tsQueryBufferSize >= 0) { +// tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; +// } + +// cfgPrintCfg(pCfg); + +// return 0; +// } +#endif \ No newline at end of file diff --git a/source/libs/config/test/CMakeLists.txt b/source/libs/config/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1e63d5025dc975b8af50fd5ef3adfe2dae2e0772 --- /dev/null +++ b/source/libs/config/test/CMakeLists.txt @@ -0,0 +1,14 @@ +enable_testing() + +aux_source_directory(. CFG_TEST_SRC) +add_executable(cfg_test ${CFG_TEST_SRC}) +target_link_libraries( + cfg_test + PUBLIC config + PUBLIC gtest_main +) + +add_test( + NAME cfg_test + COMMAND cfg_test +) diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7c70e1a8f88fc17e398c75460b0c861fcba1e4bf --- /dev/null +++ b/source/libs/config/test/cfgTest.cpp @@ -0,0 +1,147 @@ +/** + * @file cfgTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief config module tests + * @version 1.0 + * @date 2022-02-20 + * + * @copyright Copyright (c) 2022 + * + */ + +#include +#include "config.h" + +class CfgTest : public ::testing::Test { + protected: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} + + public: + void SetUp() override {} + void TearDown() override {} + void InitCfg(SConfig *pConfig); + + static const char *pConfig; +}; + +const char *CfgTest::pConfig; + +TEST_F(CfgTest, 01_Str) { + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_DEFAULT), "default"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_CFG_FILE), "cfg_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_VAR), "env_var"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ARG_LIST), "arg_list"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_API_OPTION), "api_option"); + EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); + + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_IPSTR), "ipstr"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); +} + +TEST_F(CfgTest, 02_Basic) { + SConfig *pConfig = cfgInit(); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4"), 0); + EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1"), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp"), 0); + + EXPECT_EQ(cfgGetSize(pConfig), 7); + + int32_t size = 0; + SConfigItem *pItem = cfgIterate(pConfig, NULL); + while (pItem != NULL) { + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval); + break; + case CFG_DTYPE_INT32: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->i32); + break; + case CFG_DTYPE_INT64: + printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->fval); + break; + case CFG_DTYPE_STRING: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + case CFG_DTYPE_IPSTR: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + case CFG_DTYPE_DIR: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + default: + printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); + break; + } + size++; + pItem = cfgIterate(pConfig, pItem); + } + cfgCancelIterate(pConfig, pItem); + + EXPECT_EQ(cfgGetSize(pConfig), 7); + + pItem = cfgGetItem(pConfig, "test_bool"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); + EXPECT_STREQ(pItem->name, "test_bool"); + EXPECT_EQ(pItem->bval, 0); + + pItem = cfgGetItem(pConfig, "test_int32"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); + EXPECT_STREQ(pItem->name, "test_int32"); + EXPECT_EQ(pItem->i32, 1); + + pItem = cfgGetItem(pConfig, "test_int64"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); + EXPECT_STREQ(pItem->name, "test_int64"); + EXPECT_EQ(pItem->i64, 2); + + pItem = cfgGetItem(pConfig, "test_float"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT); + EXPECT_STREQ(pItem->name, "test_float"); + EXPECT_EQ(pItem->fval, 3); + + pItem = cfgGetItem(pConfig, "test_string"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_STRING); + EXPECT_STREQ(pItem->name, "test_string"); + EXPECT_STREQ(pItem->str, "4"); + + pItem = cfgGetItem(pConfig, "test_ipstr"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_IPSTR); + EXPECT_STREQ(pItem->name, "test_ipstr"); + EXPECT_STREQ(pItem->str, "192.168.0.1"); + + pItem = cfgGetItem(pConfig, "test_dir"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); + EXPECT_STREQ(pItem->name, "test_dir"); + EXPECT_STREQ(pItem->str, "/tmp"); + + cfgCleanup(pConfig); +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 18485249b39f47faf19e25d8079016af2acd213d..4078ee9291963724ee69989021552c089239bf01 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,7 +35,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) } else { SStreamBlockScanInfo* pInfo = pOperator->info; if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) { - qError("submit msg error while set stream msg, %s" PRIx64, id); + qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5508b5ecd713016a4d228a7087d34faaadddc38f..cc05d433eb013b32abaed70290fbdeb0b7208584 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -132,7 +132,8 @@ do { \ } while (0) int32_t getMaximumIdleDurationSec() { - return tsShellActivityTimer * 2; + // todo + return 6; //tsShellActivityTimer * 2; } static int32_t getExprFunctionId(SExprInfo *pExprInfo) { @@ -4628,7 +4629,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); int32_t TENMB = 1024*1024*10; - int32_t code = createDiskbasedBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, tsTempDir); + int32_t code = createDiskbasedBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, osTempDir()); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5301,10 +5302,12 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* rpcInit.label = "EX"; rpcInit.numOfThreads = 1; rpcInit.cfp = qProcessFetchRsp; - rpcInit.sessions = tsMaxConnections; + // todo + rpcInit.sessions = 50000; //tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)"root"; - rpcInit.idleTime = tsShellActivityTimer * 1000; + // todo + rpcInit.idleTime = 6; //tsShellActivityTimer * 1000; rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index 62f86de918036dff27ef7b1d43b9d2cace16da9b..598a28b2eb10afdbf38f58e57a7667b8d7eb5a99 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -46,6 +46,7 @@ typedef struct SBuiltinFuncDefinition { FExecGetEnv getEnvFunc; FExecInit initFunc; FExecProcess processFunc; + FScalarExecProcess sprocessFunc; FExecFinalize finalizeFunc; } SBuiltinFuncDefinition; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 96e165f7886b866cee63db78c5bd4e2cf432dacf..73ce67bd2847b3f1b20769e1626514ad5a3646ee 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -36,7 +36,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .checkFunc = stubCheckAndGetResultType, .getEnvFunc = NULL, .initFunc = NULL, - .processFunc = NULL, + .sprocessFunc = NULL, .finalizeFunc = NULL } }; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 887f65a6ea7ee4d56b976fc74bb8f801fe3d0677..c6330c6015b346c03e9477efa138f769f3c7ee8d 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -71,6 +71,14 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { return TSDB_CODE_SUCCESS; } +int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return TSDB_CODE_FAILED; + } + pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc; + return TSDB_CODE_SUCCESS; +} + bool fmIsAggFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index aa3e6ba0f0664b6c781b75e6cf70818b1e037ae1..7d9cb97400732c57bbb52c1abe5a85b47d0d057d 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -25,6 +25,7 @@ #include "thash.h" #include "texpr.h" #include "tvariant.h" +#include "tdef.h" //static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { // if (pLeft->nodeType == TEXPR_COL_NODE) { @@ -94,7 +95,7 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp //non-leaf nodes, recursively traverse the expression tree in the post-root order if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) { - if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + if (pExpr->_node.optr == LOGIC_COND_TYPE_OR) { // or if (exprTreeApplyFilter(pLeft, pItem, param)) { return true; } @@ -255,7 +256,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_LIKE; + expr->_node.optr = OP_TYPE_LIKE; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1; @@ -266,7 +267,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_MATCH; + expr->_node.optr = OP_TYPE_MATCH; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN) + 1; @@ -276,7 +277,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { pVal->nLen = (int32_t)len; } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_NMATCH; + expr->_node.optr = OP_TYPE_NMATCH; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_NMATCH_LEN) + 1; @@ -286,7 +287,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { pVal->nLen = (int32_t)len; } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_IN; + expr->_node.optr = OP_TYPE_IN; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY; diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 40731adc58326da548bd53d4de6f2825efeff021..e58cdf8802d7aa48d02e87eb07d2319b60e98a0d 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -254,7 +254,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir); + int32_t ret = createDiskbasedBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, osTempDir()); if (ret != 0) { tMemBucketDestroy(pBucket); return NULL; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index b65e637a57496486919f0114dd633077f337d8d1..98bcf189b0809ff385517f3507825d378f59324a 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -55,7 +55,7 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { } else { char path[PATH_MAX] = {0}; - taosGetTmpfilePath("script", path, tsTempDir); + taosGetTmpfilePath("script", path, osTempDir()); FILE* file = fopen(path, "w+"); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 3664bcfad0dc492a2383673f346b5605f5c02637..a6cabbd43940c91fb62afe9c65fc9dc4bb4139c3 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -235,6 +235,7 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran FstState st = fstStateCreate(OneTrans); fstStateSetCommInput(&st, trn->inp); + bool null = false; uint8_t inp = fstStateCommInput(&st, &null); if (null == true) { @@ -936,6 +937,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { } void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } + void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; if (trn == NULL) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 590ff294bf8f1841fc27e4519df1d23f668adcd5..ed1ad7a374286c8251dcb719be9a5510c6eec844 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,24 +16,24 @@ #include "index_fst_automation.h" StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { - StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { + StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue)); + if (sv == NULL) { return NULL; } - nsv->kind = kind; - nsv->type = ty; + sv->kind = kind; + sv->type = ty; if (ty == FST_INT) { - nsv->val = *(int*)val; + sv->val = *(int*)val; } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); - nsv->ptr = (char*)calloc(1, len + 1); - memcpy(nsv->ptr, val, len); + sv->ptr = (char*)calloc(1, len + 1); + memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, // nsv->arr = taosArrayFromList() } - return nsv; + return sv; } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; @@ -146,11 +146,9 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (atype == AUTOMATION_ALWAYS) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTMMATION_MATCH) { } else { // add more search type @@ -160,9 +158,8 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (data != NULL) { char* src = (char*)data; size_t len = strlen(src); - dst = (char*)malloc(len * sizeof(char) + 1); + dst = (char*)calloc(1, len * sizeof(char) + 1); memcpy(dst, src, len); - dst[len] = 0; } ctx->data = dst; diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index d10f5f47b8d5e0b4980efacb78e0fd5271a187b2..0e16f57f58876369911948785ed328e028162a43 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -30,8 +30,8 @@ static void EnvInit() { taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); // init log file - snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str()); - if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) { + osSetLogDir( path.c_str()); + if (taosInitLog("tindex.idx", 1) != 0) { printf("failed to init log"); } // init index file @@ -99,6 +99,7 @@ class FstReadMemory { fstSliceDestroy(&skey); return ok; } + bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) { int64_t s = taosGetTimestampUs(); bool ok = this->Get(key, val); @@ -120,8 +121,6 @@ class FstReadMemory { printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); swsResultDestroy(rt); } - for (size_t i = 0; i < result.size(); i++) { - } std::cout << std::endl; return true; } @@ -137,7 +136,7 @@ class FstReadMemory { fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc, false); - tfCleanup(); + // tfCleanup(); } private: @@ -196,6 +195,10 @@ class TFst { } return fr->Get(k, v); } + bool Search(AutomationCtx* ctx, std::vector& result) { + // add more + return fr->Search(ctx, result); + } private: FstWriter* fw; @@ -229,5 +232,9 @@ TEST_F(FstEnv, writeNormal) { assert(fst->Get("a", &val) == false); assert(fst->Get("aa", &val) == true); assert(val == 0); + + std::vector rlt; + AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + assert(fst->Search(ctx, rlt) == true); } -TEST_F(FstEnv, writeExcpet) {} +TEST_F(FstEnv, WriteMillonrRecord) {} diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 745982e869c6bf092babd2369fd2420f9a99c2b4..a88f679655b31bdf487428f892dc61751ab338b9 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -204,7 +204,7 @@ static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time bool isSigned = false; toInteger(pToken->z, pToken->n, 10, &ts, &isSigned); } else { // parse the RFC-3339/ISO-8601 timestamp format string - if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(pToken->z, time, pToken->n, timePrec, osDaylight()) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index ef040fdff47ff925acfbdc08071147d2eb56bf74..8e29f66c574eddafaec8b0507da06b180dbcc468 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -578,7 +578,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { return DEAL_RES_ERROR; } int32_t len = trimStringCopy(pVal->literal, n, tmp); - if (taosParseTime(tmp, &pVal->datum.i, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(tmp, &pVal->datum.i, len, pVal->node.resType.precision, osDaylight()) != TSDB_CODE_SUCCESS) { tfree(tmp); generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); return DEAL_RES_ERROR; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 4fd0de3803699912c519f698189cd95640ee71b3..2647806e03ad796c94a4cda6ace4dfd8acf0ee58 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -884,7 +884,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil } assert(src->filterstr == 0 || src->filterstr == 1); - assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0)); return pFilter; } @@ -1507,45 +1507,45 @@ int32_t getTagFilterSerializeLen(SQueryStmtInfo* pQueryInfo) { uint32_t convertRelationalOperator(SToken *pToken) { switch (pToken->type) { case TK_LT: - return TSDB_RELATION_LESS; + return OP_TYPE_LOWER_THAN; case TK_LE: - return TSDB_RELATION_LESS_EQUAL; + return OP_TYPE_LOWER_EQUAL; case TK_GT: - return TSDB_RELATION_GREATER; + return OP_TYPE_GREATER_THAN; case TK_GE: - return TSDB_RELATION_GREATER_EQUAL; + return OP_TYPE_GREATER_EQUAL; case TK_NE: - return TSDB_RELATION_NOT_EQUAL; + return OP_TYPE_NOT_EQUAL; case TK_AND: - return TSDB_RELATION_AND; + return LOGIC_COND_TYPE_AND; case TK_OR: - return TSDB_RELATION_OR; + return LOGIC_COND_TYPE_OR; case TK_EQ: - return TSDB_RELATION_EQUAL; + return OP_TYPE_EQUAL; case TK_PLUS: - return TSDB_BINARY_OP_ADD; + return OP_TYPE_ADD; case TK_MINUS: - return TSDB_BINARY_OP_SUBTRACT; + return OP_TYPE_SUB; case TK_STAR: - return TSDB_BINARY_OP_MULTIPLY; + return OP_TYPE_MULTI; case TK_SLASH: case TK_DIVIDE: - return TSDB_BINARY_OP_DIVIDE; + return OP_TYPE_DIV; case TK_REM: - return TSDB_BINARY_OP_REMAINDER; + return OP_TYPE_MOD; case TK_LIKE: - return TSDB_RELATION_LIKE; + return OP_TYPE_LIKE; case TK_MATCH: - return TSDB_RELATION_MATCH; + return OP_TYPE_MATCH; case TK_NMATCH: - return TSDB_RELATION_NMATCH; + return OP_TYPE_NMATCH; case TK_ISNULL: - return TSDB_RELATION_ISNULL; + return OP_TYPE_IS_NULL; case TK_NOTNULL: - return TSDB_RELATION_NOTNULL; + return OP_TYPE_IS_NOT_NULL; case TK_IN: - return TSDB_RELATION_IN; + return OP_TYPE_IN; default: { return 0; } } } @@ -1639,7 +1639,7 @@ static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time bool isSigned = false; toInteger(pToken->z, pToken->n, 10, &ts, &isSigned); } else { // parse the RFC-3339/ISO-8601 timestamp format string - if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(pToken->z, time, pToken->n, timePrec, osDaylight()) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index b408e15abe8518261560be7ef902ed1f0cab24cd..b97176013265cf3493806603f093ac09620d1ba5 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -593,7 +593,7 @@ TEST(testCase, function_Test6) { SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pQueryInfo->exprList[1], 0); ASSERT_EQ(p2->pExpr->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(p2->pExpr->_node.optr, TSDB_BINARY_OP_ADD); + ASSERT_EQ(p2->pExpr->_node.optr, OP_TYPE_ADD); ASSERT_EQ(p2->pExpr->_node.pLeft->nodeType, TEXPR_COL_NODE); ASSERT_EQ(p2->pExpr->_node.pRight->nodeType, TEXPR_COL_NODE); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 4cd040c238931cb228ffcc4ebb6fb1fd3e781f17..b29817a3187e83c545140f44bf06f988d8b8b1a4 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -85,9 +85,11 @@ static void* pTaskQueue = NULL; int32_t initTaskQueue() { double factor = 4.0; - int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + // todo + // int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t numOfThreads = TMAX((int)(tsNumOfCores * 1.0f / factor), 2); - int32_t queueSize = tsMaxConnections * 2; + int32_t queueSize = 25000; //tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); if (NULL == pTaskQueue) { qError("failed to init task queue"); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 38890150ee26ffc1fb092a1a6047c3b2237c1631..d8df9a81c4f8a0480ce775ec8d62ffd764de2117 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -101,10 +101,8 @@ void qwtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", osLogDir()); } } diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index f51dd66ccaee7d8fcc4709532c4b3b28478d6e4a..a0d4bc680cc1742ee76dffa0b139f83bb079ff3e 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -26,6 +26,7 @@ extern "C" { #include "scalar.h" #include "querynodes.h" #include "query.h" +#include "tep.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 @@ -43,7 +44,6 @@ enum { FLD_TYPE_COLUMN = 1, FLD_TYPE_VALUE = 2, FLD_TYPE_MAX = 3, - FLD_DESC_NO_FREE = 4, FLD_DATA_NO_FREE = 8, FLD_DATA_IS_HASH = 16, }; @@ -61,11 +61,6 @@ enum { RANGE_FLG_NULL = 4, }; -enum { - FI_OPTION_NO_REWRITE = 1, - FI_OPTION_TIMESTAMP = 2, - FI_OPTION_NEED_UNIQE = 4, -}; enum { FI_STATUS_ALL = 1, @@ -107,7 +102,6 @@ typedef struct SFilterRange { typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); -typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **); typedef struct SFilterRangeCompare { @@ -206,7 +200,7 @@ typedef struct SFilterUnit { } SFilterUnit; typedef struct SFilterComUnit { - void *colData; + void *colData; // pointer to SColumnInfoData void *valData; void *valData2; uint16_t colId; @@ -264,12 +258,12 @@ typedef struct SFilterInfo { } SFilterInfo; #define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) -#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) +#define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) -#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) -#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) +#define SET_AND_OPTR(ctx, o) do {if (o == OP_TYPE_IS_NULL) { (ctx)->isnull = true; } else if (o == OP_TYPE_IS_NOT_NULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) +#define SET_OR_OPTR(ctx,o) do {if (o == OP_TYPE_IS_NULL) { (ctx)->isnull = true; } else if (o == OP_TYPE_IS_NOT_NULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) #define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true) #define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true))) @@ -289,7 +283,7 @@ typedef struct SFilterInfo { #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) -#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_NULL) +#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_UNKNOWN) #define fltFatal(...) qFatal(__VA_ARGS__) #define fltError(...) qError(__VA_ARGS__) @@ -312,7 +306,7 @@ typedef struct SFilterInfo { #define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId) #define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId) #define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnNode *)((fi)->desc))->node.resType.bytes * (ri)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGet(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) @@ -351,23 +345,10 @@ typedef struct SFilterInfo { #define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL) #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) -#if 0 -extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); -extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp); -extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp); -extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); -extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); -extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); -extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); -#else -//REMOVE THESE!!!!!!!!!!!!!!!!!!!! -#include "function.h" -#endif extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); + #ifdef __cplusplus } #endif diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 3cfea6890f20cdaf9c2cb2ab92affed68312ff2f..41ee90667e012dde68d9c7179a05b3595001182f 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -28,6 +28,8 @@ typedef struct SScalarCtx { SHashObj *pRes; /* element is SScalarParam */ } SScalarCtx; + +#define SCL_DATA_TYPE_DUMMY_HASH 9000 #define SCL_DEFAULT_OP_NUM 10 #define sclFatal(...) qFatal(__VA_ARGS__) diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 69800a54ea6e36a349b47a199287d676c4707d63..55c482874511c86ef630c189df751f6b018b71d7 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -24,7 +24,6 @@ extern "C" { typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); -bool isBinaryStringOp(int32_t op); #ifdef __cplusplus } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index b50228a3dd3676aa78c13d67100e2976f3b56c21..92e30f86f22e3f0f6393436f307828b1821ae82b 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -21,22 +21,42 @@ #include "filter.h" OptrStr gOptrStr[] = { - {TSDB_RELATION_INVALID, "invalid"}, - {TSDB_RELATION_LESS, "<"}, - {TSDB_RELATION_GREATER, ">"}, - {TSDB_RELATION_EQUAL, "="}, - {TSDB_RELATION_LESS_EQUAL, "<="}, - {TSDB_RELATION_GREATER_EQUAL, ">="}, - {TSDB_RELATION_NOT_EQUAL, "!="}, - {TSDB_RELATION_LIKE, "like"}, - {TSDB_RELATION_ISNULL, "is null"}, - {TSDB_RELATION_NOTNULL, "not null"}, - {TSDB_RELATION_IN, "in"}, - {TSDB_RELATION_AND, "and"}, - {TSDB_RELATION_OR, "or"}, - {TSDB_RELATION_NOT, "not"}, - {TSDB_RELATION_MATCH, "match"}, - {TSDB_RELATION_NMATCH, "nmatch"} + {0, "invalid"}, + {OP_TYPE_ADD, "+"}, + {OP_TYPE_SUB, "-"}, + {OP_TYPE_MULTI, "*"}, + {OP_TYPE_DIV, "/"}, + {OP_TYPE_MOD, "%"}, + + // bit operator + {OP_TYPE_BIT_AND, "&"}, + {OP_TYPE_BIT_OR, "|"}, + + // comparison operator + {OP_TYPE_GREATER_THAN, ">"}, + {OP_TYPE_GREATER_EQUAL, ">="}, + {OP_TYPE_LOWER_THAN, "<"}, + {OP_TYPE_LOWER_EQUAL, "<="}, + {OP_TYPE_EQUAL, "=="}, + {OP_TYPE_NOT_EQUAL, "!="}, + {OP_TYPE_IN, "in"}, + {OP_TYPE_NOT_IN, "not in"}, + {OP_TYPE_LIKE, "like"}, + {OP_TYPE_NOT_LIKE, "not like"}, + {OP_TYPE_MATCH, "match"}, + {OP_TYPE_NMATCH, "nmatch"}, + {OP_TYPE_IS_NULL, "is null"}, + {OP_TYPE_IS_NOT_NULL, "not null"}, + {OP_TYPE_IS_TRUE, "is true"}, + {OP_TYPE_IS_FALSE, "is false"}, + {OP_TYPE_IS_UNKNOWN, "is unknown"}, + {OP_TYPE_IS_NOT_TRUE, "not true"}, + {OP_TYPE_IS_NOT_FALSE, "not false"}, + {OP_TYPE_IS_NOT_UNKNOWN, "not unknown"}, + + // json operator + {OP_TYPE_JSON_GET_VALUE, "json get"}, + {OP_TYPE_JSON_CONTAINS, "json contains"} }; bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -110,30 +130,30 @@ rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRan int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { if (optr2) { - assert(optr2 == TSDB_RELATION_LESS || optr2 == TSDB_RELATION_LESS_EQUAL); + assert(optr2 == OP_TYPE_LOWER_THAN || optr2 == OP_TYPE_LOWER_EQUAL); - if (optr == TSDB_RELATION_GREATER) { - if (optr2 == TSDB_RELATION_LESS) { + if (optr == OP_TYPE_GREATER_THAN) { + if (optr2 == OP_TYPE_LOWER_THAN) { return 0; } return 1; } - if (optr2 == TSDB_RELATION_LESS) { + if (optr2 == OP_TYPE_LOWER_THAN) { return 2; } return 3; } else { switch (optr) { - case TSDB_RELATION_GREATER: + case OP_TYPE_GREATER_THAN: return 4; - case TSDB_RELATION_GREATER_EQUAL: + case OP_TYPE_GREATER_EQUAL: return 5; - case TSDB_RELATION_LESS: + case OP_TYPE_LOWER_THAN: return 6; - case TSDB_RELATION_LESS_EQUAL: + case OP_TYPE_LOWER_EQUAL: return 7; default: break; @@ -154,7 +174,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t comparFn = 0; - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -177,7 +197,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } } - if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -210,17 +230,17 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = 7; - } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ comparFn = 26; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_IN) { comparFn = 8; - } else if (optr == TSDB_RELATION_NOT_IN) { + } else if (optr == OP_TYPE_NOT_IN) { comparFn = 25; } else { /* normal relational comparFn */ comparFn = 6; @@ -230,17 +250,17 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { + } else if (optr == OP_TYPE_LIKE) { comparFn = 9; - } else if (optr == TSDB_RELATION_LIKE) { + } else if (optr == OP_TYPE_LIKE) { comparFn = 27; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_IN) { comparFn = 8; - } else if (optr == TSDB_RELATION_NOT_IN) { + } else if (optr == OP_TYPE_NOT_IN) { comparFn = 25; } else { comparFn = 10; @@ -383,7 +403,7 @@ int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull int32_t filterAddRangeOptr(void* h, uint8_t raOptr, int32_t optr, bool *empty, bool *all) { SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { SET_AND_OPTR(ctx, raOptr); if (CHK_AND_OPTR(ctx) || (raOptr == FILTER_DUMMY_EMPTY_OPTR)) { FILTER_SET_FLAG(ctx->status, MR_ST_EMPTY); @@ -407,8 +427,8 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { if (ctx->rs == NULL) { if ((FILTER_GET_FLAG(ctx->status, MR_ST_START) == 0) - || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == TSDB_RELATION_AND)) - || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == TSDB_RELATION_OR))) { + || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == LOGIC_COND_TYPE_AND)) + || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == LOGIC_COND_TYPE_OR))) { APPEND_RANGE(ctx, ctx->rs, ra); FILTER_SET_FLAG(ctx->status, MR_ST_START); } @@ -420,7 +440,7 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { SFilterRangeNode *rn = NULL; int32_t cr = 0; - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { while (r != NULL) { cr = ctx->pCompareFunc(&r->ra.s, &ra->e); if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { @@ -532,7 +552,7 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { if (notnull) { bool all = false; FREE_FROM_RANGE(ctx, ctx->rs); - filterAddRangeOptr(h, TSDB_RELATION_NOTNULL, optr, NULL, &all); + filterAddRangeOptr(h, OP_TYPE_IS_NOT_NULL, optr, NULL, &all); if (all) { FILTER_SET_FLAG(ctx->status, MR_ST_ALL); } @@ -563,7 +583,7 @@ int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - assert(optr == TSDB_RELATION_OR); + assert(optr == LOGIC_COND_TYPE_OR); if (sctx->rs == NULL) { return TSDB_CODE_SUCCESS; @@ -614,13 +634,13 @@ int32_t filterFinishRange(void* h) { return TSDB_CODE_SUCCESS; } - if (FILTER_GET_FLAG(ctx->options, FI_OPTION_TIMESTAMP)) { + if (FILTER_GET_FLAG(ctx->options, FLT_OPTION_TIMESTAMP)) { SFilterRangeNode *r = ctx->rs; SFilterRangeNode *rn = NULL; while (r && r->next) { int64_t tmp = 1; - operateVal(&tmp, &r->ra.e, &tmp, TSDB_BINARY_OP_ADD, ctx->type); + operateVal(&tmp, &r->ra.e, &tmp, OP_TYPE_ADD, ctx->type); if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { rn = r->next; SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); @@ -685,14 +705,14 @@ int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, SFilterRangeCtx *src = (SFilterRangeCtx *)sctx; if (src->isnull){ - filterAddRangeOptr(ctx, TSDB_RELATION_ISNULL, optr, empty, all); + filterAddRangeOptr(ctx, OP_TYPE_IS_NULL, optr, empty, all); if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { *all = true; } } if (src->notnull) { - filterAddRangeOptr(ctx, TSDB_RELATION_NOTNULL, optr, empty, all); + filterAddRangeOptr(ctx, OP_TYPE_IS_NOT_NULL, optr, empty, all); if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { *all = true; } @@ -701,7 +721,7 @@ int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, if (src->isrange) { filterAddRangeOptr(ctx, 0, optr, empty, all); - if (!(optr == TSDB_RELATION_OR && ctx->notnull)) { + if (!(optr == LOGIC_COND_TYPE_OR && ctx->notnull)) { filterAddRangeCtx(ctx, src, optr); } @@ -764,13 +784,26 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { int32_t rightSize = (int32_t)taosArrayGetSize(right); if (taosArrayGetSize(left) <= 0) { - fltDebug("empty group"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + if (taosArrayGetSize(right) <= 0) { + fltError("both groups are empty"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SFilterGroup *gp = NULL; + while (gp = (SFilterGroup *)taosArrayPop(right)) { + taosArrayPush(group, gp); + } + + return TSDB_CODE_SUCCESS; } if (taosArrayGetSize(right) <= 0) { - fltDebug("empty group"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SFilterGroup *gp = NULL; + while (gp = (SFilterGroup *)taosArrayPop(left)) { + taosArrayPush(group, gp); + } + + return TSDB_CODE_SUCCESS; } for (int32_t l = 0; l < leftSize; ++l) { @@ -825,7 +858,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, if (*num > 0) { if (type == FLD_TYPE_COLUMN) { idx = filterGetFiledByDesc(&info->fields[type], type, desc); - } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { idx = filterGetFiledByData(info, type, *data, dataLen); } } @@ -847,7 +880,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, ++(*num); - if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { if (info->pctx.valHash == NULL) { info->pctx.valHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_VALUE_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); } @@ -855,10 +888,6 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); } } else { - if (freeIfExists) { - tfree(desc); - } - if (data && freeIfExists) { tfree(*data); } @@ -873,7 +902,6 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); - FILTER_SET_FLAG(field->flag, FLD_DESC_NO_FREE); FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); return TSDB_CODE_SUCCESS; @@ -886,7 +914,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE) { + if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } @@ -907,7 +935,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f } int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) { - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { if (info->pctx.unitHash == NULL) { info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); } else { @@ -940,8 +968,10 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); } else { - if(optr != TSDB_RELATION_ISNULL && optr != TSDB_RELATION_NOTNULL && optr != FILTER_DUMMY_EMPTY_OPTR){ - return -1; + int32_t paramNum = scalarGetOperatorParamNum(optr); + if (1 != paramNum) { + fltError("invalid right field in unit, operator:%s, rightType:%d", gOptrStr[optr].str, u->right.type); + return TSDB_CODE_QRY_APP_ERROR; } } @@ -952,7 +982,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi *uidx = info->unitNum; - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { int64_t v = 0; FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); @@ -988,7 +1018,6 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) { SNodeListNode *listNode = (SNodeListNode *)node->pRight; - void *fdata = NULL; SListCell *cell = listNode->pNodeList->pHead; SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; @@ -1008,14 +1037,16 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { len = tDataTypes[type].bytes; - filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); + filterAddField(info, NULL, &out.data, FLD_TYPE_VALUE, &right, len, true); - filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx); SFilterGroup fgroup = {0}; filterAddUnitToGroup(&fgroup, uidx); taosArrayPush(group, &fgroup); + + cell = cell->pNext; } } else { filterAddFieldFromNode(info, node->pRight, &right); @@ -1035,16 +1066,15 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) { SFilterFieldId left, right, *pright = &right; int32_t type = FILTER_UNIT_DATA_TYPE(u); - uint16_t flag = FLD_DESC_NO_FREE; + uint16_t flag = 0; filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); if (u->right.type == FLD_TYPE_VALUE) { void *data = FILTER_UNIT_VAL_DATA(src, u); if (IS_VAR_DATA_TYPE(type)) { - if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { + if (FILTER_UNIT_OPTR(u) == OP_TYPE_IN) { filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. t = FILTER_GET_FIELD(dst, right); @@ -1087,17 +1117,17 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { if (ctx->isnull) { assert(ctx->notnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } if (ctx->notnull) { assert(ctx->isnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } @@ -1119,7 +1149,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } else { @@ -1130,8 +1160,8 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SIMPLE_COPY_VALUES(data2, &ra->e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } @@ -1141,7 +1171,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1149,7 +1179,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1164,7 +1194,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan assert(ctx->isnull || ctx->notnull || ctx->isrange); if (ctx->isnull) { - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); taosArrayPush(res, g); } @@ -1173,7 +1203,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan assert(!ctx->isrange); memset(g, 0, sizeof(*g)); - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); taosArrayPush(res, g); } @@ -1195,7 +1225,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } else { void *data = malloc(sizeof(int64_t)); @@ -1205,8 +1235,8 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SIMPLE_COPY_VALUES(data2, &r->ra.e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, uidx); filterAddUnitToGroup(g, uidx); } @@ -1221,7 +1251,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1229,7 +1259,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1262,7 +1292,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { SArray* preGroup = NULL; SArray* newGroup = NULL; SArray* resGroup = NULL; - ENodeType nType = nodeType(nType); + ENodeType nType = nodeType(pNode); SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext; if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { @@ -1280,16 +1310,18 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); taosArrayDestroyEx(newGroup, filterFreeGroup); + newGroup = NULL; taosArrayDestroyEx(preGroup, filterFreeGroup); preGroup = resGroup; + resGroup = NULL; + + cell = cell->pNext; } - taosArrayAddAll(ctx->group, resGroup); + taosArrayAddAll(ctx->group, preGroup); - taosArrayDestroyEx(newGroup, filterFreeGroup); - taosArrayDestroyEx(preGroup, filterFreeGroup); - taosArrayDestroyEx(resGroup, filterFreeGroup); + taosArrayDestroy(preGroup); return DEAL_RES_IGNORE_CHILD; } @@ -1299,11 +1331,15 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { for (int32_t i = 0; i < node->pParameterList->length; ++i) { nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); FLT_ERR_JRET(ctx->code); + + cell = cell->pNext; } return DEAL_RES_IGNORE_CHILD; } + ctx->code = TSDB_CODE_QRY_APP_ERROR; + fltError("invalid condition type, type:%d", node->condType); return DEAL_RES_ERROR; @@ -1312,7 +1348,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { if (QUERY_NODE_OPERATOR == nType) { FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group)); - return DEAL_RES_CONTINUE; + return DEAL_RES_IGNORE_CHILD; } fltError("invalid node type for filter, type:%d", nodeType(pNode)); @@ -1448,11 +1484,11 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SColumnNode *refNode = (SColumnNode *)left->desc; - if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_NMATCH){ + if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){ len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); } - if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); char *data = right->data; if (IS_VAR_DATA_TYPE(type)) { @@ -1467,11 +1503,11 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_NMATCH){ + if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS){ sprintf(str + strlen(str), "[%d][%d] %s [", refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr2].str); } - if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); char *data = right->data; if (IS_VAR_DATA_TYPE(type)) { @@ -1619,14 +1655,6 @@ void filterFreeField(SFilterField* field, int32_t type) { return; } - if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) { - if (type == FLD_TYPE_VALUE) { - taosVariantDestroy(field->desc); - } - - tfree(field->desc); - } - if (!FILTER_GET_FLAG(field->flag, FLD_DATA_NO_FREE)) { if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { taosHashCleanup(field->data); @@ -1688,15 +1716,15 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { uint8_t optr = FILTER_UNIT_OPTR(unit); switch (optr) { - case TSDB_RELATION_GREATER: - case TSDB_RELATION_GREATER_EQUAL: - unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : TSDB_RELATION_NOTNULL; + case OP_TYPE_GREATER_THAN: + case OP_TYPE_GREATER_EQUAL: + unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : OP_TYPE_IS_NOT_NULL; break; - case TSDB_RELATION_LESS: - case TSDB_RELATION_LESS_EQUAL: - unit->compare.optr = (extInfo > 0) ? TSDB_RELATION_NOTNULL : FILTER_DUMMY_EMPTY_OPTR; + case OP_TYPE_LOWER_THAN: + case OP_TYPE_LOWER_EQUAL: + unit->compare.optr = (extInfo > 0) ? OP_TYPE_IS_NOT_NULL : FILTER_DUMMY_EMPTY_OPTR; break; - case TSDB_RELATION_EQUAL: + case OP_TYPE_EQUAL: unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; break; default: @@ -1706,75 +1734,12 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { return TSDB_CODE_SUCCESS; } -int32_t fltGenerateSetFromList(void **data, void *pNode, uint32_t type) { - SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); - if (NULL == pObj) { - fltError("taosHashInit failed, size:%d", 256); - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); - - int32_t code = 0; - SNodeListNode *nodeList = (SNodeListNode *)pNode; - SListCell *cell = nodeList->pNodeList->pHead; - SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; - int8_t dummy = 0; - int32_t bufLen = 60; - out.data = malloc(bufLen); - int32_t len = 0; - void *buf = NULL; - - for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { - SValueNode *valueNode = (SValueNode *)cell->pNode; - - if (valueNode->node.resType.type != type) { - in.type = valueNode->node.resType.type; - in.bytes = valueNode->node.resType.bytes; - in.data = nodesGetValueFromNode(valueNode); - - code = vectorConvertImpl(&in, &out); - if (code) { - fltError("convert from %d to %d failed", in.type, out.type); - FLT_ERR_JRET(code); - } - - if (IS_VAR_DATA_TYPE(type)) { - len = varDataLen(out.data); - } else { - len = tDataTypes[type].bytes; - } - - buf = out.data; - } else { - buf = nodesGetValueFromNode(valueNode); - len = valueNode->node.resType.bytes; - } - - if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) { - fltError("taosHashPut failed"); - FLT_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - - tfree(out.data); - *data = pObj; - - return TSDB_CODE_SUCCESS; - -_return: - - tfree(out.data); - taosHashCleanup(pObj); - - FLT_RET(code); -} int32_t fltInitValFieldData(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit* unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { - assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); + assert(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1); continue; } @@ -1792,8 +1757,8 @@ int32_t fltInitValFieldData(SFilterInfo *info) { continue; } - if (unit->compare.optr == TSDB_RELATION_IN) { - FLT_ERR_RET(fltGenerateSetFromList((void **)&fi->data, fi->desc, type)); + if (unit->compare.optr == OP_TYPE_IN) { + FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type)); if (fi->data == NULL) { fltError("failed to convert in param"); FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -1830,24 +1795,26 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } if(type != TSDB_DATA_TYPE_JSON){ - bool converted = false; - char extInfo = 0; - SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; - SScalarParam out = {.data = fi->data, .num = 1, .type = type}; - if (vectorConvertImpl(&in, &out)) { - qError("convert value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; + if (dType->type == type) { + assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + } else { + SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; + SScalarParam out = {.data = fi->data, .num = 1, .type = type}; + if (vectorConvertImpl(&in, &out)) { + qError("convert value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; + } } } // match/nmatch for nchar type need convert from ucs4 to mbs if(type == TSDB_DATA_TYPE_NCHAR && - (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ + (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)){ char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); if (len < 0){ qError("filterInitValFieldData taosUcs4ToMbs error 1"); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_APP_ERROR; } varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); @@ -1862,34 +1829,40 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) int32_t ret = func(left, right); switch (optr) { - case TSDB_RELATION_EQUAL: { + case OP_TYPE_EQUAL: { return ret == 0; } - case TSDB_RELATION_NOT_EQUAL: { + case OP_TYPE_NOT_EQUAL: { return ret != 0; } - case TSDB_RELATION_GREATER_EQUAL: { + case OP_TYPE_GREATER_EQUAL: { return ret >= 0; } - case TSDB_RELATION_GREATER: { + case OP_TYPE_GREATER_THAN: { return ret > 0; } - case TSDB_RELATION_LESS_EQUAL: { + case OP_TYPE_LOWER_EQUAL: { return ret <= 0; } - case TSDB_RELATION_LESS: { + case OP_TYPE_LOWER_THAN: { return ret < 0; } - case TSDB_RELATION_LIKE: { + case OP_TYPE_LIKE: { return ret == 0; } - case TSDB_RELATION_MATCH: { + case OP_TYPE_NOT_LIKE: { return ret == 0; } - case TSDB_RELATION_NMATCH: { + case OP_TYPE_MATCH: { return ret == 0; } - case TSDB_RELATION_IN: { + case OP_TYPE_NMATCH: { + return ret == 0; + } + case OP_TYPE_IN: { + return ret == 1; + } + case OP_TYPE_NOT_IN: { return ret == 1; } @@ -1909,25 +1882,25 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c int64_t tmp = 0; switch (uoptr) { - case TSDB_RELATION_GREATER: + case OP_TYPE_GREATER_THAN: SIMPLE_COPY_VALUES(&ra.s, val); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_EXCLUDE); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_GREATER_EQUAL: + case OP_TYPE_GREATER_EQUAL: SIMPLE_COPY_VALUES(&ra.s, val); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_LESS: + case OP_TYPE_LOWER_THAN: SIMPLE_COPY_VALUES(&ra.e, val); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_EXCLUDE); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_LESS_EQUAL: + case OP_TYPE_LOWER_EQUAL: SIMPLE_COPY_VALUES(&ra.e, val); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_NOT_EQUAL: + case OP_TYPE_NOT_EQUAL: assert(type == TSDB_DATA_TYPE_BOOL); if (GET_INT8_VAL(val)) { SIMPLE_COPY_VALUES(&ra.s, &tmp); @@ -1938,7 +1911,7 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c SIMPLE_COPY_VALUES(&ra.e, &tmp); } break; - case TSDB_RELATION_EQUAL: + case OP_TYPE_EQUAL: SIMPLE_COPY_VALUES(&ra.s, val); SIMPLE_COPY_VALUES(&ra.e, val); break; @@ -1992,15 +1965,15 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colI SFilterUnit* u = taosArrayGetP(colArray, i); uint8_t optr = FILTER_UNIT_OPTR(u); - filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); + filterAddRangeOptr(ctx, optr, LOGIC_COND_TYPE_AND, empty, NULL); FLT_CHK_JMP(*empty); if (!FILTER_NO_MERGE_OPTR(optr)) { - filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); + filterAddUnitRange(info, u, ctx, LOGIC_COND_TYPE_AND); FLT_CHK_JMP(MR_EMPTY_RES(ctx)); } - if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){ - gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL; + if(FILTER_UNIT_OPTR(u) == OP_TYPE_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){ + gRes->colInfo[colIdx].optr = OP_TYPE_EQUAL; SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u)); } } @@ -2120,7 +2093,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool } // for long in operation - if (gRes1->colInfo[idx1].optr == TSDB_RELATION_EQUAL && gRes2->colInfo[idx2].optr == TSDB_RELATION_EQUAL) { + if (gRes1->colInfo[idx1].optr == OP_TYPE_EQUAL && gRes2->colInfo[idx2].optr == OP_TYPE_EQUAL) { SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info; if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){ *conflict = true; @@ -2196,7 +2169,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++merNum; - filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); + filterMergeTwoGroupsImpl(info, &ctx, LOGIC_COND_TYPE_OR, idx1, *gRes1, *gRes2, NULL, all); FLT_CHK_JMP(*all); @@ -2410,14 +2383,14 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum oinfo.colRangeNum = 0; oinfo.colRange = NULL; - FILTER_SET_FLAG(info->options, FI_OPTION_NEED_UNIQE); + FILTER_SET_FLAG(info->options, FLT_OPTION_NEED_UNIQE); filterInitUnitsFields(info); for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; - optr = (res->colNum > 1) ? TSDB_RELATION_AND : TSDB_RELATION_OR; + optr = (res->colNum > 1) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR; SFilterGroup ng = {0}; @@ -2514,7 +2487,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ assert(colInfo->type == RANGE_TYPE_MR_CTX); bool all = false; - filterSourceRangeFromCtx(info->colRange[m], colInfo->info, TSDB_RELATION_OR, NULL, &all); + filterSourceRangeFromCtx(info->colRange[m], colInfo->info, LOGIC_COND_TYPE_OR, NULL, &all); if (all) { filterFreeRangeCtx(info->colRange[m]); info->colRange[m] = NULL; @@ -2597,7 +2570,8 @@ int32_t filterUpdateComUnits(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; - info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); + SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, unit); + info->cunits[i].colData = col->data; } return TSDB_CODE_SUCCESS; @@ -2629,20 +2603,20 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } if (pDataStatis[index].numOfNull <= 0) { - if (cunit->optr == TSDB_RELATION_ISNULL) { + if (cunit->optr == OP_TYPE_IS_NULL) { info->blkUnitRes[k] = -1; rmUnit = 1; continue; } - if (cunit->optr == TSDB_RELATION_NOTNULL) { + if (cunit->optr == OP_TYPE_IS_NOT_NULL) { info->blkUnitRes[k] = 1; rmUnit = 1; continue; } } else { if (pDataStatis[index].numOfNull == numOfRows) { - if (cunit->optr == TSDB_RELATION_ISNULL) { + if (cunit->optr == OP_TYPE_IS_NULL) { info->blkUnitRes[k] = 1; rmUnit = 1; continue; @@ -2654,9 +2628,9 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } } - if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL - || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH - || cunit->optr == TSDB_RELATION_NOT_EQUAL) { + if (cunit->optr == OP_TYPE_IS_NULL || cunit->optr == OP_TYPE_IS_NOT_NULL + || cunit->optr == OP_TYPE_IN || cunit->optr == OP_TYPE_LIKE || cunit->optr == OP_TYPE_MATCH + || cunit->optr == OP_TYPE_NOT_EQUAL) { continue; } @@ -2686,8 +2660,8 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 info->blkUnitRes[k] = 1; rmUnit = 1; } else if ((!minRes) && (!maxRes)) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS_EQUAL, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER_EQUAL, maxVal, cunit->valData2); + minRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_LOWER_EQUAL, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_GREATER_EQUAL, maxVal, cunit->valData2); if (minRes && maxRes) { continue; @@ -2704,9 +2678,9 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 info->blkUnitRes[k] = 1; rmUnit = 1; } else if ((!minRes) && (!maxRes)) { - if (cunit->optr == TSDB_RELATION_EQUAL) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS, maxVal, cunit->valData); + if (cunit->optr == OP_TYPE_EQUAL) { + minRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_GREATER_THAN, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_LOWER_THAN, maxVal, cunit->valData); if (minRes || maxRes) { info->blkUnitRes[k] = -1; rmUnit = 1; @@ -2802,7 +2776,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint32_t unitNum = *(unitIdx++); for (uint32_t u = 0; u < unitNum; ++u) { SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; + void *colData = colDataGet((SColumnInfoData *)cunit->colData, i); //if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); @@ -2810,11 +2784,11 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint8_t optr = cunit->optr; if (isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { - if (optr == TSDB_RELATION_NOTNULL) { + if (optr == OP_TYPE_IS_NOT_NULL) { (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { + } else if (optr == OP_TYPE_IS_NULL) { (*p)[i] = 0; } else if (cunit->rfunc >= 0) { (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); @@ -2900,7 +2874,7 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ if (!colData){ // for json->'key' is null (*p)[i] = 1; @@ -2934,7 +2908,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ if (!colData) { // for json->'key' is not null @@ -2961,7 +2935,6 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; uint16_t dataSize = info->cunits[0].dataSize; - char *colData = (char *)info->cunits[0].colData; rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc]; void *valData = info->cunits[0].valData; void *valData2 = info->cunits[0].valData2; @@ -2975,10 +2948,11 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD *p = calloc(numOfRows, sizeof(int8_t)); } - for (int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { + void *colData = colDataGet((SColumnInfoData *)info->cunits[0].colData, i); + if (colData == NULL || isNull(colData, info->cunits[0].dataType)) { all = false; - colData += dataSize; continue; } @@ -2987,8 +2961,6 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD if ((*p)[i] == 0) { all = false; } - - colData += dataSize; } return all; @@ -3008,7 +2980,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) { (*p)[i] = 0; all = false; @@ -3016,7 +2988,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa } // match/nmatch for nchar type need convert from ucs4 to mbs - if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){ + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){ char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ @@ -3059,7 +3031,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg for (uint32_t u = 0; u < group->unitNum; ++u) { uint32_t uidx = group->unitIdxs[u]; SFilterComUnit *cunit = &info->cunits[uidx]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; + void *colData = colDataGet((SColumnInfoData *)(cunit->colData), i); //if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); @@ -3067,16 +3039,16 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg uint8_t optr = cunit->optr; if (colData == NULL || isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { - if (optr == TSDB_RELATION_NOTNULL) { + if (optr == OP_TYPE_IS_NOT_NULL) { (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { + } else if (optr == OP_TYPE_IS_NULL) { (*p)[i] = 0; } else if (cunit->rfunc >= 0) { (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { - if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){ + if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ @@ -3129,12 +3101,12 @@ int32_t filterSetExecFunc(SFilterInfo *info) { return TSDB_CODE_SUCCESS; } - if (info->units[0].compare.optr == TSDB_RELATION_ISNULL) { + if (info->units[0].compare.optr == OP_TYPE_IS_NULL) { info->func = filterExecuteImplIsNull; return TSDB_CODE_SUCCESS; } - if (info->units[0].compare.optr == TSDB_RELATION_NOTNULL) { + if (info->units[0].compare.optr == OP_TYPE_IS_NOT_NULL) { info->func = filterExecuteImplNotNull; return TSDB_CODE_SUCCESS; } @@ -3230,7 +3202,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { FLT_ERR_JRET(fltInitValFieldData(info)); - if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { + if (!FILTER_GET_FLAG(info->options, FLT_OPTION_NO_REWRITE)) { filterDumpInfoToString(info, "Before preprocess", 0); FLT_ERR_JRET(filterPreprocess(info)); @@ -3251,7 +3223,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { _return: - qInfo("No filter, code:%d", code); + qInfo("init from node failed, code:%d", code); return code; } @@ -3348,8 +3320,8 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { SFilterRange ra = {0}; - SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); - SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FLT_OPTION_TIMESTAMP); + SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FLT_OPTION_TIMESTAMP); SFilterRangeCtx *cur = NULL; int32_t num = 0; int32_t optr = 0; @@ -3360,10 +3332,10 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { SFilterGroup *group = &info->groups[i]; if (group->unitNum > 1) { cur = tmpc; - optr = TSDB_RELATION_AND; + optr = LOGIC_COND_TYPE_AND; } else { cur = prev; - optr = TSDB_RELATION_OR; + optr = LOGIC_COND_TYPE_OR; } for (uint32_t u = 0; u < group->unitNum; ++u) { @@ -3372,21 +3344,14 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { uint8_t raOptr = FILTER_UNIT_OPTR(unit); - filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); + filterAddRangeOptr(cur, raOptr, LOGIC_COND_TYPE_AND, &empty, NULL); FLT_CHK_JMP(empty); if (FILTER_NO_MERGE_OPTR(raOptr)) { continue; } - - SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - void *s = FILTER_GET_VAL_FIELD_DATA(right); - void *e = FILTER_GET_VAL_FIELD_DATA(right) + tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; - SIMPLE_COPY_VALUES(&ra.s, s); - SIMPLE_COPY_VALUES(&ra.e, e); - - filterAddRange(cur, &ra, optr); + filterAddUnitRange(info, unit, cur, optr); } if (cur->notnull) { @@ -3395,7 +3360,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { } if (group->unitNum > 1) { - filterSourceRangeFromCtx(prev, cur, TSDB_RELATION_OR, &empty, &all); + filterSourceRangeFromCtx(prev, cur, LOGIC_COND_TYPE_OR, &empty, &all); filterResetRangeCtx(cur); if (all) { break; @@ -3522,11 +3487,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (stat->scalarMode) { - return DEAL_RES_CONTINUE; - } - - if (QUERY_NODE_VALUE == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } @@ -3544,7 +3505,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { if (NULL == node->pRight) { if (scalarGetOperatorParamNum(node->opType) > 1) { - fltError("invalid operator, pRight:%p, type:%d", node->pRight, nodeType(node)); + fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); stat->code = TSDB_CODE_QRY_APP_ERROR; return DEAL_RES_ERROR; } @@ -3553,6 +3514,12 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } + + if (OP_TYPE_IS_TRUE == node->opType || OP_TYPE_IS_FALSE == node->opType || OP_TYPE_IS_UNKNOWN == node->opType + || OP_TYPE_IS_NOT_TRUE == node->opType || OP_TYPE_IS_NOT_FALSE == node->opType || OP_TYPE_IS_NOT_UNKNOWN == node->opType) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } } else { if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { stat->scalarMode = true; @@ -3609,18 +3576,19 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { - + //TODO + return TSDB_CODE_SUCCESS; } -int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { +int32_t fltGetDataFromColId(void *param, int32_t id, void **data) { int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; for (int32_t j = 0; j < numOfCols; ++j) { SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); if (id == pColInfo->info.colId) { - *data = pColInfo->pData; + *data = pColInfo; break; } } @@ -3628,13 +3596,28 @@ int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { return TSDB_CODE_SUCCESS; } +int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) { + int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; + SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; + if (id < 0 || id >= numOfCols || id >= taosArrayGetSize(pDataBlock)) { + fltError("invalid slot id, id:%d, numOfCols:%d, arraySize:%d", id, numOfCols, (int32_t)taosArrayGetSize(pDataBlock)); + return TSDB_CODE_QRY_APP_ERROR; + } + + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, id); + *data = pColInfo; -int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { - return fltSetColFieldDataImpl(info, param, fp, false); + return TSDB_CODE_SUCCESS; } -int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { - return fltSetColFieldDataImpl(info, param, fp, true); + + +int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param) { + return fltSetColFieldDataImpl(info, param, fltGetDataFromSlotId, false); +} + +int32_t filterSetDataFromColId(SFilterInfo *info, void *param) { + return fltSetColFieldDataImpl(info, param, fltGetDataFromColId, true); } @@ -3662,6 +3645,8 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) SFltTreeStat stat = {0}; FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); + info->scalarMode = stat.scalarMode; + if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); } else { @@ -3680,13 +3665,21 @@ _return: FLT_RET(code); } -FORCE_INLINE bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { if (info->scalarMode) { SScalarParam output = {0}; FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output)); *p = output.data; - return TSDB_CODE_SUCCESS; + + int8_t *r = output.data; + for (int32_t i = 0; i < output.num; ++i) { + if (0 == *(r+i)) { + return false; + } + } + + return true; } return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 095a88e04078f17d48720263fd3eab7794af4301..e4ad45700cf8e6c9506a0e8fafad0801dc88ac72 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -7,13 +7,87 @@ #include "sclInt.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { - if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type) { + if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type + || OP_TYPE_IS_FALSE == type || OP_TYPE_IS_NOT_FALSE == type || OP_TYPE_IS_UNKNOWN == type || OP_TYPE_IS_NOT_UNKNOWN == type) { return 1; } return 2; } +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); + if (NULL == pObj) { + sclError("taosHashInit failed, size:%d", 256); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); + + int32_t code = 0; + SNodeListNode *nodeList = (SNodeListNode *)pNode; + SListCell *cell = nodeList->pNodeList->pHead; + SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; + int8_t dummy = 0; + int32_t bufLen = 60; + out.data = malloc(bufLen); + int32_t len = 0; + void *buf = NULL; + + for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + + if (valueNode->node.resType.type != type) { + in.type = valueNode->node.resType.type; + in.bytes = valueNode->node.resType.bytes; + in.data = nodesGetValueFromNode(valueNode); + + code = vectorConvertImpl(&in, &out); + if (code) { + sclError("convert from %d to %d failed", in.type, out.type); + SCL_ERR_JRET(code); + } + + if (IS_VAR_DATA_TYPE(type)) { + len = varDataLen(out.data); + buf = varDataVal(out.data); + } else { + len = tDataTypes[type].bytes; + buf = out.data; + } + } else { + buf = nodesGetValueFromNode(valueNode); + if (IS_VAR_DATA_TYPE(type)) { + len = varDataLen(buf); + buf = varDataVal(buf); + } else { + len = valueNode->node.resType.bytes; + buf = out.data; + } + } + + if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) { + sclError("taosHashPut failed"); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + cell = cell->pNext; + } + + tfree(out.data); + *data = pObj; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(out.data); + taosHashCleanup(pObj); + + SCL_RET(code); +} + + void sclFreeRes(SHashObj *res) { SScalarParam *p = NULL; void *pIter = taosHashIterate(res, NULL); @@ -42,12 +116,22 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t param->num = 1; param->type = valueNode->node.resType.type; param->bytes = valueNode->node.resType.bytes; + param->colData = false; break; } case QUERY_NODE_NODE_LIST: { SNodeListNode *nodeList = (SNodeListNode *)node; - //TODO BUILD HASH + if (nodeList->pNodeList->length <= 0) { + sclError("invalid length in nodeList, length:%d", nodeList->pNodeList->length); + SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SCL_ERR_RET(scalarGenerateSetFromList(¶m->data, node, nodeList->dataType.type)); + param->num = 1; + param->type = SCL_DATA_TYPE_DUMMY_HASH; + param->colData = false; + break; } case QUERY_NODE_COLUMN: { @@ -63,7 +147,14 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); - param->data = columnData->pData; + if (IS_VAR_DATA_TYPE(columnData->info.type)) { + param->data = columnData; + param->colData = true; + } else { + param->data = columnData->pData; + param->colData = false; + } + param->num = ctx->pSrc->info.rows; param->type = columnData->info.type; param->bytes = columnData->info.bytes; @@ -250,11 +341,15 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); output->type = node->node.resType.type; + output->bytes = sizeof(bool); + output->num = rowNum; output->data = calloc(rowNum, sizeof(bool)); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + void *data = output->data; bool value = false; @@ -277,6 +372,8 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o sclParamMoveNext(params, node->pParameterList->length); } + output->data = data; + return TSDB_CODE_SUCCESS; _return: @@ -293,6 +390,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); output->type = node->node.resType.type; + output->num = rowNum; + output->bytes = tDataTypes[output->type].bytes; output->data = calloc(rowNum, tDataTypes[output->type].bytes); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); @@ -304,17 +403,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t paramNum = scalarGetOperatorParamNum(node->opType); SScalarParam* pLeft = ¶ms[0]; SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; - - for (int32_t i = 0; i < rowNum; ++i) { - - OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); - - sclParamMoveNext(output, 1); - sclParamMoveNext(pLeft, 1); - if (pRight) { - sclParamMoveNext(pRight, 1); - } - } + + OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); return TSDB_CODE_SUCCESS; @@ -429,7 +519,7 @@ EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } @@ -511,10 +601,10 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) { EDealRes sclCalcWalker(SNode* pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) { return DEAL_RES_CONTINUE; } - + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { return sclWalkFunction(pNode, pContext); } @@ -527,7 +617,7 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) { return sclWalkOperator(pNode, pContext); } - sclError("invalid node type for calculating constants, type:%d", nodeType(pNode)); + sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); SScalarCtx *ctx = (SScalarCtx *)pContext; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index ab2cc8b0569acec445271c6852195f80f019d883..fd900afcda78a255cde001ffba8c29d69b826b36 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -268,9 +268,6 @@ static void setScalarFuncParam(SScalarParam* param, int32_t type, int32_t bytes, param->data = pInput; } -bool isStringOp(int32_t op) { - return op == TSDB_BINARY_OP_CONCAT; -} #if 0 int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 5b2ec7902ce4876a886ae604ad9786430c523780..85af663313eeb64669d4066274202308a320297c 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -21,6 +21,9 @@ #include "querynodes.h" #include "filterInt.h" #include "query.h" +#include "sclInt.h" +#include "tep.h" +#include "filter.h" //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); @@ -93,6 +96,7 @@ double getVectorDoubleValue_FLOAT(void *src, int32_t index) { double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { return (double)*((double *)src + index); } + _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { _getDoubleValue_fn_t p = NULL; if(srcType==TSDB_DATA_TYPE_TINYINT) { @@ -218,6 +222,12 @@ void* getVectorValueAddr_FLOAT(void *src, int32_t index) { void* getVectorValueAddr_DOUBLE(void *src, int32_t index) { return (void*)((double *)src + index); } +void* getVectorValueAddr_default(void *src, int32_t index) { + return src; +} +void* getVectorValueAddr_VAR(void *src, int32_t index) { + return colDataGet((SColumnInfoData *)src, index); +} _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { _getValueAddr_fn_t p = NULL; @@ -241,8 +251,12 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { p = getVectorValueAddr_FLOAT; }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { p = getVectorValueAddr_DOUBLE; + }else if(srcType==TSDB_DATA_TYPE_BINARY) { + p = getVectorValueAddr_VAR; + }else if(srcType==TSDB_DATA_TYPE_NCHAR) { + p = getVectorValueAddr_VAR; }else { - assert(0); + p = getVectorValueAddr_default; } return p; } @@ -258,31 +272,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { switch (outType) { case TSDB_DATA_TYPE_BOOL: - if (inType == TSDB_DATA_TYPE_BINARY) { - for (int32_t i = 0; i < pIn->num; ++i) { - GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input)); - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += sizeof(bool); - } - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - for (int32_t i = 0; i < pIn->num; ++i) { - GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input)); - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } - } - break; + case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: @@ -291,21 +281,26 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) + 1; char *tmp = malloc(bufSize); if (NULL == tmp) { + sclError("malloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); } - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; } @@ -315,25 +310,30 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; char *tmp = calloc(1, bufSize); if (NULL == tmp) { + sclError("calloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - qError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } - tmp[len] = 0; - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + } input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; @@ -351,6 +351,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { } } break; + case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: @@ -358,20 +359,25 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) + 1; char *tmp = malloc(bufSize); if (NULL == tmp) { + sclError("malloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); } - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; } @@ -381,25 +387,30 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; char *tmp = calloc(1, bufSize); if (NULL == tmp) { + sclError("calloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - qError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } - tmp[len] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + } input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; @@ -408,9 +419,13 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { tfree(tmp); } else { for (int32_t i = 0; i < pIn->num; ++i) { - uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, input); - SET_TYPED_DATA(output, outType, value); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + } input += tDataTypes[inType].bytes; output += tDataTypes[outType].bytes; @@ -423,21 +438,26 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) + 1; char *tmp = malloc(bufSize); if (NULL == tmp) { + sclError("malloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); } - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); - input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; } @@ -447,25 +467,30 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; char *tmp = calloc(1, bufSize); if (NULL == tmp) { + sclError("calloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - qError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } - tmp[len] = 0; - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); + } input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; @@ -474,9 +499,13 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { tfree(tmp); } else { for (int32_t i = 0; i < pIn->num; ++i) { - int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, input); - SET_TYPED_DATA(output, outType, value); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + } input += tDataTypes[inType].bytes; output += tDataTypes[outType].bytes; @@ -484,7 +513,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { } break; default: - qError("invalid convert output type:%d", outType); + sclError("invalid convert output type:%d", outType); return TSDB_CODE_QRY_APP_ERROR; } @@ -532,6 +561,10 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p return TSDB_CODE_SUCCESS; } + if (SCL_DATA_TYPE_DUMMY_HASH == pLeft->type || SCL_DATA_TYPE_DUMMY_HASH == pRight->type) { + return TSDB_CODE_SUCCESS; + } + SScalarParam *param1 = NULL, *paramOut1 = NULL; SScalarParam *param2 = NULL, *paramOut2 = NULL; int32_t code = 0; @@ -595,6 +628,46 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double *output=(double*)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -628,12 +701,55 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double *output=(double*)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -666,11 +782,54 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double *output=(double*)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -704,12 +863,55 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double *output=(double*)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -750,12 +952,55 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double * output = (double *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -822,6 +1067,9 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { @@ -879,6 +1127,46 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + int64_t *output=(int64_t *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -912,12 +1200,55 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + int64_t *output=(int64_t *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -951,6 +1282,9 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } @@ -961,10 +1295,23 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int bool res = false; bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + _getValueAddr_fn_t getVectorValueAddrFnRight = NULL; + + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } + + if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->colData) { + getVectorValueAddrFnRight = getVectorValueAddr_default; + } else { + getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + } + - if (pLeft->num == pRight->num) { + if (pLeft->num == pRight->num) { for (; i < pRight->num && i >= 0; i += step, output += 1) { if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { @@ -992,7 +1339,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); } } else if (pRight->num == 1) { - void *rightData = getVectorValueAddrFnRight(pRight->data, 0); + void *rightData = getVectorValueAddrFnRight(pRight->data, 0); for (; i >= 0 && i < pLeft->num; i += step, output += 1) { if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) { @@ -1030,55 +1377,55 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t param2 = pRight; } - vectorCompareImpl(param1, param2, out, _ord, TSDB_RELATION_GREATER); + vectorCompareImpl(param1, param2, out, _ord, optr); } void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_THAN); } void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER_EQUAL); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_EQUAL); } void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_THAN); } void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS_EQUAL); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_EQUAL); } void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_EQUAL); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_EQUAL); } void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_EQUAL); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_EQUAL); } void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_IN); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_IN); } void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_IN); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_IN); } void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LIKE); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LIKE); } void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_LIKE); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_LIKE); } void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_MATCH); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_MATCH); } void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NMATCH); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NMATCH); } void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { @@ -1087,7 +1434,13 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t bool res = false; bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } for (; i >= 0 && i < pLeft->num; i += step, output += 1) { if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { @@ -1107,7 +1460,13 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t bool res = false; bool *output = (bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } for (; i >= 0 && i < pLeft->num; i += step, output += 1) { if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { @@ -1121,6 +1480,17 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t } } +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + SScalarParam output = {.data = out, .num = pLeft->num, .type = TSDB_DATA_TYPE_BOOL}; + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + vectorConvertImpl(pLeft, &output); +} + _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { switch (binFunctionId) { @@ -1166,12 +1536,12 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorBitAnd; case OP_TYPE_BIT_OR: return vectorBitOr; + case OP_TYPE_IS_TRUE: + return vectorIsTrue; default: assert(0); return NULL; } } -bool isBinaryStringOp(int32_t op) { - return op == TSDB_BINARY_OP_CONCAT; -} + diff --git a/source/libs/scalar/test/CMakeLists.txt b/source/libs/scalar/test/CMakeLists.txt index a9af1ece30ac51a8059f03f4592b0a37cbc9724c..caaf86264c579196c9e5b8988a6c633ef1a70d2c 100644 --- a/source/libs/scalar/test/CMakeLists.txt +++ b/source/libs/scalar/test/CMakeLists.txt @@ -1,18 +1,4 @@ +enable_testing() -MESSAGE(STATUS "build scalar unit test") - -# GoogleTest requires at least C++11 -SET(CMAKE_CXX_STANDARD 11) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - -ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) -TARGET_LINK_LIBRARIES( - scalarTest - PUBLIC os util common gtest qcom function nodes -) - -TARGET_INCLUDE_DIRECTORIES( - scalarTest - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" - PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" -) +add_subdirectory(filter) +add_subdirectory(scalar) diff --git a/source/libs/scalar/test/filter/CMakeLists.txt b/source/libs/scalar/test/filter/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b738285dedcea2c2a01d406ed76facc987e3162e --- /dev/null +++ b/source/libs/scalar/test/filter/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build filter unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(filterTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + filterTest + PUBLIC os util common gtest qcom function nodes scalar +) + +TARGET_INCLUDE_DIRECTORIES( + filterTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" +) diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d27fe2c25ff7dbeb93dccae165ae4db042443ca2 --- /dev/null +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -0,0 +1,1175 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "stub.h" +#include "addr_any.h" +#include "scalar.h" +#include "nodes.h" +#include "tlog.h" +#include "filter.h" + +namespace { + +int64_t flttLeftV = 21, flttRightV = 10; +double flttLeftVd = 21.0, flttRightVd = 10.0; + +void flttInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", osLogDir()); + } +} + + +void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *vnode = (SValueNode *)node; + vnode->node.resType.type = dataType; + + if (IS_VAR_DATA_TYPE(dataType)) { + vnode->datum.p = (char *)malloc(varDataTLen(value)); + varDataCopy(vnode->datum.p, value); + vnode->node.resType.bytes = varDataLen(value); + } else { + vnode->node.resType.bytes = tDataTypes[dataType].bytes; + assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); + } + + *pNode = (SNode *)vnode; +} + +void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); + SColumnRefNode *rnode = (SColumnRefNode *)node; + rnode->dataType.type = dataType; + rnode->dataType.bytes = dataBytes; + rnode->tupleId = 0; + + if (NULL == block) { + rnode->slotId = 2; + rnode->columnId = 55; + *pNode = (SNode *)rnode; + + return; + } + + if (NULL == *block) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 3; + res->info.rows = rowNum; + res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < 2; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = TSDB_DATA_TYPE_NULL; + idata.info.bytes = 10; + idata.info.colId = 0; + + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + } + + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55; + idata.pData = (char *)value; + if (IS_VAR_DATA_TYPE(dataType)) { + idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); + for (int32_t i = 0; i < rowNum; ++i) { + idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + } + } + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = 2; + rnode->columnId = 55; + + *block = res; + } else { + SSDataBlock *res = *block; + + int32_t idx = taosArrayGetSize(res->pDataBlock); + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55 + idx; + idata.pData = (char *)value; + taosArrayPush(res->pDataBlock, &idata); + + res->info.numOfCols++; + + rnode->slotId = idx; + rnode->columnId = 55 + idx; + } + + *pNode = (SNode *)rnode; +} + +void flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { + SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + onode->pLeft = pLeft; + onode->pRight = pRight; + + *pNode = (SNode *)onode; +} + +void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodesMakeList(); + for (int32_t i = 0; i < nodeNum; ++i) { + nodesListAppend(onode->pParameterList, nodeList[i]); + } + + *pNode = (SNode *)onode; +} + +void flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeList *nodeList) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodeList; + + *pNode = (SNode *)onode; +} + +void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { + SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *lnode = (SNodeListNode *)node; + lnode->dataType.type = resType; + lnode->pNodeList = list; + + *pNode = (SNode *)lnode; +} + + +} + +TEST(timerangeTest, greater) { + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + bool eRes[5] = {false, false, true, true, true}; + SScalarParam res = {0}; + int64_t tsmall = 222, tbig = 333; + flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode1, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + ASSERT_EQ(code, 0); + STimeWindow win = {0}; + code = filterGetTimeRange(filter, &win); + ASSERT_EQ(code, 0); + ASSERT_EQ(win.skey, tsmall); + ASSERT_EQ(win.ekey, INT64_MAX); +} + +TEST(timerangeTest, greater_and_lower) { + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + bool eRes[5] = {false, false, true, true, true}; + SScalarParam res = {0}; + int64_t tsmall = 222, tbig = 333; + flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + SNode *list[2] = {0}; + list[0] = opNode1; + list[1] = opNode2; + + flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + ASSERT_EQ(code, 0); + STimeWindow win = {0}; + code = filterGetTimeRange(filter, &win); + ASSERT_EQ(code, 0); + ASSERT_EQ(win.skey, tsmall); + ASSERT_EQ(win.ekey, tbig); +} + + +TEST(columnTest, smallint_column_greater_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 2.5; + int8_t eRes[5] = {0, 0, 1, 1, 1}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + stat.colId = ((SColumnRefNode *)pLeft)->columnId; + stat.max = 10; + stat.min = 5; + stat.numOfNull = 0; + bool keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 1; + stat.min = -1; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 10; + stat.min = 5; + stat.numOfNull = rowNum; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(columnTest, int_column_greater_smallint_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv[5]= {1, 3, 5, 7, 9}; + int16_t rightv= 4; + int8_t eRes[5] = {0, 0, 1, 1, 1}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + stat.colId = ((SColumnRefNode *)pLeft)->columnId; + stat.max = 10; + stat.min = 5; + stat.numOfNull = 0; + bool keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 1; + stat.min = -1; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, false); + + stat.max = 10; + stat.min = 5; + stat.numOfNull = rowNum; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, false); + + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(columnTest, int_column_in_double_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + int32_t leftv[5] = {1, 2, 3, 4, 5}; + double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + SNodeList* list = nodesMakeList(); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv2); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv3); + nodesListAppend(list, pRight); + flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + + +} + + + +TEST(columnTest, binary_column_in_binary_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + char leftv[5][5]= {0}; + char rightv[3][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a' + i; + leftv[i][3] = 'b' + i; + leftv[i][4] = '0' + i; + varDataSetLen(leftv[i], 3); + } + for (int32_t i = 0; i < 2; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'b' + i; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + for (int32_t i = 2; i < 3; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'a' + i; + rightv[i][4] = 'a' + i; + varDataSetLen(rightv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SNodeList* list = nodesMakeList(); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[1]); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[2]); + nodesListAppend(list, pRight); + flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); + flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(columnTest, binary_column_like_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + char rightv[64] = {0}; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, false, true, false, true}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a'; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + sprintf(&rightv[2], "%s", "__0"); + varDataSetLen(rightv, strlen(&rightv[2])); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(columnTest, binary_column_is_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, false, false, false, true}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_not_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, true, true, true, false}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +TEST(opTest, smallint_column_greater_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5] = {1, -6, -2, 11, 101}; + int32_t rightv[5]= {0, -5, -4, 23, 100}; + bool eRes[5] = {true, false, true, false, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(opTest, smallint_value_add_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv = 1; + int16_t rightv[5]= {0, -1, -4, -1, 100}; + bool eRes[5] = {true, false, true, false, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +TEST(opTest, bigint_column_multi_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int64_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + bool eRes[5] = {false, true, true, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(opTest, smallint_column_and_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + bool eRes[5] = {false, false, true, false, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(opTest, smallint_column_or_float_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 0, 4, 5}; + float rightv[5]= {2.0, 3.0, 0, 5.2, 6.0}; + bool eRes[5] = {true, true, false, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +TEST(opTest, smallint_column_or_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {0, 2, 3, 0, -1}; + double rightv= 10.2; + bool eRes[5] = {true, true, true, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, true); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(opTest, binary_column_is_true) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, true, false, true, false}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(filterModelogicTest, diff_columns_and_or_and) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1= 3, rightv2= 3; + int8_t eRes[8] = {1, 1, 0, 0, 1, 1, 1, 1}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + list = nodesMakeList(); + + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(filterModelogicTest, same_column_and_or_and) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}; + int32_t rightv1= 3, rightv2= 0, rightv3 = 2, rightv4 = -2; + int8_t eRes[8] = {1, 1, 0, 0, 0, 1, 1, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + list = nodesMakeList(); + + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(filterModelogicTest, diff_columns_or_and_or) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1= 3, rightv2= 3; + int8_t eRes[8] = {1, 0, 1, 1, 0, 0, 0, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(filterModelogicTest, same_column_or_and_or) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}; + int32_t rightv1= 3, rightv2= 0, rightv3 = 2, rightv4 = -2; + int8_t eRes[8] = {0, 0, 0, 1, 1, 1, 1, 1}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +TEST(scalarModelogicTest, diff_columns_or_and_or) { + flttInitLogFile(); + + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8] = {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1[8]= {5, 8, 2, -3, 9,-7,10, 0}, rightv2[8]= {-3, 5, 8, 2, -9, 11, -4, 0}; + int8_t eRes[8] = {0, 1, 1, 0, 0, 1, 0, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColRefNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColRefNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColRefNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColRefNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +int main(int argc, char** argv) { + srand(time(NULL)); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#pragma GCC diagnostic pop diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..9dfbf3b5e882277e5568dbd7d0360964d75122ad --- /dev/null +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scalar unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + scalarTest + PUBLIC os util common gtest qcom function nodes scalar +) + +TARGET_INCLUDE_DIRECTORIES( + scalarTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" +) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..de2c7f587468ac36ca06db602967cadc0136d181 --- /dev/null +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -0,0 +1,1181 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "stub.h" +#include "addr_any.h" +#include "scalar.h" +#include "nodes.h" +#include "tlog.h" + +namespace { + +int64_t scltLeftV = 21, scltRightV = 10; +double scltLeftVd = 21.0, scltRightVd = 10.0; + +void scltInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", osLogDir()); + } +} + + +void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *vnode = (SValueNode *)node; + vnode->node.resType.type = dataType; + + if (IS_VAR_DATA_TYPE(dataType)) { + vnode->datum.p = (char *)malloc(varDataTLen(value)); + varDataCopy(vnode->datum.p, value); + vnode->node.resType.bytes = varDataLen(value); + } else { + vnode->node.resType.bytes = tDataTypes[dataType].bytes; + assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); + } + + *pNode = (SNode *)vnode; +} + +void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); + SColumnRefNode *rnode = (SColumnRefNode *)node; + rnode->dataType.type = dataType; + rnode->dataType.bytes = dataBytes; + rnode->tupleId = 0; + + if (NULL == *block) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 3; + res->info.rows = rowNum; + res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < 2; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = TSDB_DATA_TYPE_NULL; + idata.info.bytes = 10; + idata.info.colId = 0; + + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + } + + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55; + idata.pData = (char *)value; + if (IS_VAR_DATA_TYPE(dataType)) { + idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); + for (int32_t i = 0; i < rowNum; ++i) { + idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + } + } + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = 2; + rnode->columnId = 55; + + *block = res; + } else { + SSDataBlock *res = *block; + + int32_t idx = taosArrayGetSize(res->pDataBlock); + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55 + idx; + idata.pData = (char *)value; + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = idx; + rnode->columnId = 55 + idx; + } + + *pNode = (SNode *)rnode; +} + +void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { + SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + onode->pLeft = pLeft; + onode->pRight = pRight; + + *pNode = (SNode *)onode; +} + + +void scltMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { + SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *lnode = (SNodeListNode *)node; + lnode->dataType.type = resType; + lnode->pNodeList = list; + + *pNode = (SNode *)lnode; +} + + +void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodesMakeList(); + for (int32_t i = 0; i < nodeNum; ++i) { + nodesListAppend(onode->pParameterList, nodeList[i]); + } + + *pNode = (SNode *)onode; +} + + +} + +TEST(constantTest, bigint_add_bigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BIGINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(v->datum.d, (scltLeftV + scltRightV)); +} + +TEST(constantTest, double_sub_bigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_DOUBLE, &scltLeftVd); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_SUB, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(v->datum.d, (scltLeftVd - scltRightV)); +} + +TEST(constantTest, tinyint_and_smallint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_TINYINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.i, (int64_t)scltLeftV & (int64_t)scltRightV); +} + +TEST(constantTest, bigint_or_double) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BIGINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &scltRightVd); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.i, (int64_t)scltLeftV | (int64_t)scltRightVd); +} + +TEST(constantTest, int_or_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char binaryStr[64] = {0}; + sprintf(&binaryStr[2], "%d", scltRightV); + varDataSetLen(binaryStr, strlen(&binaryStr[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, binaryStr); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.b, scltLeftV | scltRightV); +} + + +TEST(constantTest, int_greater_double) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &scltRightVd); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); +} + +TEST(constantTest, int_greater_equal_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char binaryStr[64] = {0}; + sprintf(&binaryStr[2], "%d", scltRightV); + varDataSetLen(binaryStr, strlen(&binaryStr[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, binaryStr); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); +} + +TEST(constantTest, tinyint_lower_ubigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_TINYINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV < scltRightV); +} + +TEST(constantTest, usmallint_lower_equal_ubigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_USMALLINT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv <= rightv); +} + +TEST(constantTest, int_equal_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv == rightv); +} + +TEST(constantTest, int_equal_smallint2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 0, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv == rightv); +} + +TEST(constantTest, int_not_equal_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv != rightv); +} + +TEST(constantTest, int_not_equal_smallint2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 0, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv != rightv); +} + + + +TEST(constantTest, int_in_smallint1) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 1, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_in_smallint2) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 4, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_not_in_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 1, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_NOT_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_not_in_smallint2) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 4, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_NOT_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_like_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a_c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_like_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "ac"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_like_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a%c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_like_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "ac"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_match_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", ".*"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_match_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "abc.+"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_match_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a[1-9]c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NMATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_not_match_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a[ab]c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NMATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_is_null1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_is_null2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_is_not_null1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_is_not_null2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_add_int_is_true1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_add_int_is_true2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = -1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + + +TEST(constantTest, int_greater_int_is_true1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_greater_int_is_true2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 0; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, greater_and_lower) { + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v4); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + + + +TEST(columnTest, smallint_value_add_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv = 1; + int16_t rightv[5]= {0, -5, -4, 23, 100}; + double eRes[5] = {1.0, -4, -3, 24, 101}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, bigint_column_multi_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int64_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + double eRes[5] = {0, 2, 6, 12, 20}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_and_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + int64_t eRes[5] = {0, 0, 2, 0, 4}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_or_float_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0}; + int64_t eRes[5] = {3, 3, 7, 5, 7}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_or_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 10.2; + int64_t eRes[5] = {11, 10, 11, 14, 15}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_greater_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 2.5; + bool eRes[5] = {false, false, true, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, int_column_in_double_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + int32_t leftv[5] = {1, 2, 3, 4, 5}; + double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_in_binary_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + char leftv[5][5]= {0}; + char rightv[3][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a' + i; + leftv[i][3] = 'b' + i; + leftv[i][4] = '0' + i; + varDataSetLen(leftv[i], 3); + } + for (int32_t i = 0; i < 2; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'b' + i; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + for (int32_t i = 2; i < 3; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'a' + i; + rightv[i][4] = 'a' + i; + varDataSetLen(rightv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[1]); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[2]); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_like_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + char rightv[64] = {0}; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, false, true, false, true}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a'; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + sprintf(&rightv[2], "%s", "__0"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_true) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, true, false, true, false}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, false, false, false, true}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_not_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, true, true, true, false}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, greater_and_lower) { + SNode *pcol1 = NULL, *pcol2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + SNode *list[2] = {0}; + int16_t v1[5]= {1, 2, 3, 4, 5}; + int32_t v2[5]= {5, 1, 4, 2, 6}; + int64_t v3[5]= {1, 2, 3, 4, 5}; + int32_t v4[5]= {5, 3, 4, 2, 6}; + bool eRes[5] = {false, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(v1)/sizeof(v1[0]); + scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); + scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); + scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int16_t), rowNum, v3); + scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v4); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculate(logicNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + + + +int main(int argc, char** argv) { + srand(time(NULL)); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#pragma GCC diagnostic pop diff --git a/source/libs/scalar/test/scalarTests.cpp b/source/libs/scalar/test/scalarTests.cpp deleted file mode 100644 index 38fe072f9e487583fcc55101acedf21ff42ea96b..0000000000000000000000000000000000000000 --- a/source/libs/scalar/test/scalarTests.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" -#pragma GCC diagnostic ignored "-Wsign-compare" -#pragma GCC diagnostic ignored "-Wformat" -#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" -#pragma GCC diagnostic ignored "-Wpointer-arith" - -#include "os.h" - -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "stub.h" -#include "addr_any.h" -#include "scalar.h" - -namespace { - -} - -TEST(scalarTest, func) { - -} - - -int main(int argc, char** argv) { - srand(time(NULL)); - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -#pragma GCC diagnostic pop diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6bfff0197c80debdcb8c9bcf920c3742c9f11776..70684412ee28d88a32d67f47301453dacd84c02b 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -66,10 +66,8 @@ void schtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", osLogDir()); } } diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index a36b671eb459268c1fcf51ca80ac21a88d050e37..d080db753d525362ecd8cca464f17f6d80fbd19e 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -66,6 +66,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + bool (*pfp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 72c1ff689374873794265fcb83908aae41c571be..cfd01bf3be7c4b0256dde3a3669b88dbaf69fb40 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -42,6 +42,8 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; +int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default + SHashObj *tsFqdnHash; #ifndef USE_UV @@ -143,6 +145,11 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; + +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; +uint32_t tsVersion = 0; + // static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 @@ -222,7 +229,9 @@ static void rpcInitImp(void) { tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); } -int32_t rpcInit(void) { +int32_t rpcInit(SRpcCfg *pCfg) { + tsRpcTimer = pCfg->rpcTimer; + tsRpcMaxTime = pCfg->rpcMaxTime; pthread_once(&tsRpcInitOnce, rpcInitImp); return 0; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 48c15ca286e6202be5d71ca7791272b0a07ce3f3..09aee6c8bcad1255173f9912ff1ddb83e4fab02a 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -29,7 +29,12 @@ void* rpcOpen(const SRpcInit* pInit) { if (pInit->label) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1); } + + // register callback handle pRpc->cfp = pInit->cfp; + pRpc->afp = pInit->afp; + pRpc->pfp = pInit->pfp; + if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } else { @@ -107,7 +112,7 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } -int32_t rpcInit(void) { +int32_t rpcInit(SRpcCfg* pCfg) { // impl later return 0; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8312c0217c13d40a43b57c82dc22ff04d5ddcd65..d5fa4c8c479494b82d97224af013f9b30cf0e961 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -134,8 +134,7 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP || - rpcMsg.msgType == TDMT_VND_RES_READY_RSP) { + if (pRpc->pfp != NULL && (pRpc->pfp)(pRpc->parent, rpcMsg.msgType)) { rpcMsg.handle = conn; conn->persist = 1; tDebug("client conn %p persist by app", conn); @@ -185,18 +184,13 @@ static void clientHandleExcept(SCliConn* pConn) { clientConnDestroy(pConn, true); return; } - SCliMsg* pMsg = pConn->data; - - tmsg_t msgType = TDMT_MND_CONNECT; - if (pMsg != NULL) { - msgType = pMsg->msg.msgType; - } + SCliMsg* pMsg = pConn->data; STransConnCtx* pCtx = pMsg->ctx; SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcMsg.msgType = msgType + 1; + rpcMsg.msgType = pMsg->msg.msgType + 1; if (pConn->push != NULL && pConn->ctnRdCnt != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); @@ -445,7 +439,7 @@ static void clientConnCb(uv_connect_t* req, int status) { addrlen = sizeof(pConn->locaddr); uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); - tTrace("client conn %p create", pConn); + tTrace("client conn %p connect to server successfully", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -524,6 +518,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { struct sockaddr_in addr; uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect + tTrace("client conn %p try to connect to %s:%d", conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 9fca371bf30588f7bdd50f5a3cc452e000bd03eb..f0db054797b936b8fde9fa3b241c8dbff909bb90 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -413,11 +413,6 @@ void uvWorkerAsyncCb(uv_async_t* handle) { } else { uvStartSendResp(msg); } - // uv_buf_t wb; - // uvPrepareSendData(msg, &wb); - // uv_timer_stop(conn->pTimer); - - // uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } static void uvAcceptAsyncCb(uv_async_t* async) { @@ -490,7 +485,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -730,14 +724,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); tDebug("send quit msg to work thread"); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } void taosCloseServer(void* arg) { @@ -774,19 +763,12 @@ void rpcSendResponse(const SRpcMsg* pMsg) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); - tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { SSrvConn* pConn = thandle; - // struct sockaddr* pPeerName = &pConn->peername; struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index f1aadafacc23f8bfd166589604789c88f7d0ecfd..4842a0c800b53a87f8f6713bb1fd045adaa9083c 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 0bcc47383b7160b92c0c5efb10406576a8a63cb9..0ccfde16ce461b6db18250e9896603794c181d73 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index cc6a63d3cd7277ced2dc6575dd8389b639bd124a..bcdf32bf6aaae04b4e526a9a6e09395f454d1be6 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 65170d4abb2745a469dfda3e4146c2ea85405b33..26a02eb05b7b704e32025731238453ffc3daece9 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) { rpcInit.label = "APP"; rpcInit.numOfThreads = 1; rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; + rpcInit.idleTime = 3000; //tsShellActivityTimer*1000; rpcInit.user = "michael"; rpcInit.secret = secret; rpcInit.ckey = "key"; @@ -148,7 +148,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index d1a587f4e5539a45001cafa28e58f134f9bcdad3..4337cd242cab7fb56f4bb6a237562bb6d6d36509 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index c5d7f5664a4665529e9068836a846518aa6abf2c..b7ef296b9deae24cb950ec5e20dfbe6083931bf6 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -170,7 +170,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index d1fefe2c727f8544ddc401bfbc8ed147009ffc93..8a8a6de5e12895c1d161e59541e11039c872a8cc 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -148,7 +148,6 @@ class TransObj { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; @@ -156,9 +155,8 @@ class TransObj { taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str()); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + osSetLogDir(path.c_str()); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } cli = new Client; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 8999646f6a7ad8b61b53f51e67d60610c0ee2028..c464073e5faeb6a9a08da886268f469189b2fbf1 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -71,7 +71,7 @@ int32_t taosMkDir(const char *dirname) { return code; } -void taosRemoveOldFiles(char *dirname, int32_t keepDays) { +void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { DIR *dir = opendir(dirname); if (dir == NULL) return; @@ -114,7 +114,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) { rmdir(dirname); } -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen) { +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordexp_t full_path; if (0 != wordexp(dirname, &full_path, 0)) { //printf("failed to expand path:%s since %s", dirname, strerror(errno)); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 7bc5e0a7fcfa759444d0d6d445a9b2eb1175f8e1..6fad7e4f3265fa65a106b022f84cb403b9e6a387 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -13,54 +13,100 @@ * along with this program. If not, see . */ - #define _DEFAULT_SOURCE -#include "os.h" +#include "osEnv.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +extern void taosWinSocketInit(); +char configDir[PATH_MAX] = {0}; + +typedef struct SOsEnv { + char dataDir[PATH_MAX]; + char logDir[PATH_MAX]; + char tempDir[PATH_MAX]; + SDiskSpace dataSpace; + SDiskSpace logSpace; + SDiskSpace tempSpace; + char osName[16]; + char timezone[TD_TIMEZONE_LEN]; + char locale[TD_LOCALE_LEN]; + char charset[TD_CHARSET_LEN]; + int8_t daylight; +} SOsEnv; + +static SOsEnv env = {0}; + +SOsEnv *osEnv() { return &env; } + +void osInitImp() { + taosGetSystemLocale(env.locale, env.charset); + taosGetSystemTimezone(env.timezone); + osSetTimezone(env.timezone); +} + +void osUpdate() { + if (env.logDir[0] != 0) { + taosGetDiskSize(env.logDir, &env.logSpace.size); + } + if (env.dataDir[0] != 0) { + taosGetDiskSize(env.dataDir, &env.dataSpace.size); + } + if (env.tempDir[0] != 0) { + taosGetDiskSize(env.tempDir, &env.tempSpace.size); + } +} + +bool osLogSpaceAvailable() { return env.logSpace.reserved <= env.logSpace.size.avail; } +int8_t osDaylight() { return env.daylight; } -char tsOsName[10] = "Windows"; -char configDir[PATH_MAX] = "C:/TDengine/cfg"; -char tsDataDir[PATH_MAX] = "C:/TDengine/data"; -char tsLogDir[PATH_MAX] = "C:/TDengine/log"; -char tsScriptDir[PATH_MAX] = "C:/TDengine/script"; -char tsTempDir[PATH_MAX] = "C:\\Windows\\Temp"; +const char *osLogDir() { return env.logDir; } +const char *osTempDir() { return env.tempDir; } +const char *osDataDir() { return env.dataDir; } +const char *osName() { return env.osName; } +const char *osTimezone() { return env.timezone; } +const char *osLocale() { return env.locale; } +const char *osCharset() { return env.charset; } -extern taosWinSocketInit(); +void osSetLogDir(const char *logDir) { tstrncpy(env.logDir, logDir, PATH_MAX); } +void osSetTempDir(const char *tempDir) { tstrncpy(env.tempDir, tempDir, PATH_MAX); } +void osSetDataDir(const char *dataDir) { tstrncpy(env.dataDir, dataDir, PATH_MAX); } +void osSetLogReservedSpace(float sizeInGB) { env.logSpace.reserved = sizeInGB; } +void osSetTempReservedSpace(float sizeInGB) { env.tempSpace.reserved = sizeInGB; } +void osSetDataReservedSpace(float sizeInGB) { env.dataSpace.reserved = sizeInGB; } +void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, env.timezone, &env.daylight); } void osInit() { + srand(taosSafeRand()); + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) taosWinSocketInit(); const char *tmpDir = getenv("tmp"); if (tmpDir == NULL) { tmpDir = getenv("temp"); } - if (tmpDir != NULL) { - strcpy(tsTempDir, tmpDir); + strcpy(env.tempDir, tmpDir); } -} -#elif defined(_TD_DARWIN_64) - -char tsOsName[10] = "Darwin"; -char configDir[PATH_MAX] = "/usr/local/etc/taos"; -char tsDataDir[PATH_MAX] = "/usr/local/var/lib/taos"; -char tsLogDir[PATH_MAX] = "/usr/local/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/usr/local/etc/taos"; -char tsTempDir[PATH_MAX] = "/tmp/taosd"; + strcpy(configDir, "C:\\TDengine\\cfg"); + strcpy(env.dataDir, "C:\\TDengine\\data"); + strcpy(env.logDir, "C:\\TDengine\\log"); + strcpy(env.tempDir, "C:\\Windows\\Temp"); + strcpy(env.osName, "Windows"); -void osInit() {} +#elif defined(_TD_DARWIN_64) + strcpy(configDir, "/tmp/taosd"); + strcpy(env.dataDir, "/usr/local/var/lib/taos"); + strcpy(env.logDir, "/usr/local/var/log/taos"); + strcpy(env.tempDir, "/usr/local/etc/taos"); + strcpy(env.osName, "Darwin"); #else - -char tsOsName[10] = "Linux"; -char configDir[PATH_MAX] = "/etc/taos"; -char tsDataDir[PATH_MAX] = "/var/lib/taos"; -char tsLogDir[PATH_MAX] = "/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/etc/taos"; -char tsTempDir[PATH_MAX] = "/tmp/"; - -void osInit() {} + strcpy(configDir, "/etc/taos"); + strcpy(env.dataDir, "/var/lib/taos"); + strcpy(env.logDir, "/var/log/taos"); + strcpy(env.tempDir, "/tmp"); + strcpy(env.osName, "Linux"); #endif +} \ No newline at end of file diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c new file mode 100644 index 0000000000000000000000000000000000000000..e006a337a0817a9f94d0d694ead331c20d4a1dff --- /dev/null +++ b/source/os/src/osLocale.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "osLocale.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + {"utf8", "UTF-8"}, + {"936", "CP936"}, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} + +/** + * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of + * both the TDengine Server and the Client may be interrupted. + * + * In case that the setLocale failed to be executed, the right charset needs to be set. + */ +void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { + char *locale = setlocale(LC_CTYPE, inLocale); + + // default locale or user specified locale is not valid, abort launch + if (inLocale == NULL) { + printf("Invalid locale:%s, please set the valid locale in config file\n", inLocale); + } + + if (strlen(inCharSet) == 0) { + printf("failed to get charset, please set the valid charset in config file\n"); + exit(-1); + } + + if (!taosValidateEncodec(inCharSet)) { + printf("Invalid charset:%s, please set the valid charset in config file", inCharSet); + exit(-1); + } +} + +void taosGetSystemLocale(char *outLocale, char *outCharset) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *locale = setlocale(LC_CTYPE, "chs"); + if (locale != NULL) { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + } + strcpy(outCharset, "cp936"); + +#elif defined(_TD_DARWIN_64) + /* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + /* if user does not specify the charset, extract it from locale */ + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#else + /* + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + // if user does not specify the charset, extract it from locale + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#endif +} diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index e12f9493b3159e85593f1723ab018ff047614851..8afb4ae8b71483172b6505c8179f8038655b66c5 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -150,7 +150,7 @@ int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER)) -uint32_t taosInetAddr(char *ipAddr) { return inet_addr(ipAddr); } +uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); } const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } @@ -240,7 +240,7 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #ifdef _MSC_VER //#if _MSC_VER >= 1900 -uint32_t taosInetAddr(char *ipAddr) { +uint32_t taosInetAddr(const char *ipAddr) { uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { diff --git a/source/os/src/osString.c b/source/os/src/osString.c index f714fd0dc4d26cf9f5c1391b34c6277b971cd74d..88ea4b3e1521103b4827c7b5646ec2ed92b556e6 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -18,24 +18,6 @@ #include #include -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} int64_t taosStr2int64(const char *str) { char *endptr = NULL; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index cae1b18b3c3dc876d9a7b2b54b1e4735ec417e4c..4a8cc5ecbc92656bd372af9684039339ccca33c8 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -21,10 +21,6 @@ int64_t tsPageSize = 0; int64_t tsOpenMax = 0; int64_t tsStreamMax = 0; int32_t tsNumOfCores = 1; -char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; -char tsLocale[TSDB_LOCALE_LEN] = {0}; -char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) /* @@ -89,25 +85,6 @@ bool taosGetProcMemory(float *memoryUsedMB) { return true; } -static void taosGetSystemTimezone() { - // get and set default timezone - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); - } else { - strcpy(tsTimezone, tz); - } -} - -static void taosGetSystemLocale() { - // get and set default locale - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - } - - strcpy(tsCharset, "cp936"); -} int32_t taosGetCpuCores() { SYSTEM_INFO info; @@ -134,7 +111,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); return 0; } else { - //printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + //printf("failed to get disk size, dataDir:%s errno:%s", osDataDir(), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -200,8 +177,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemTimezone(); - taosGetSystemLocale(); } void taosKillSystem() { @@ -273,103 +248,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; } #include #include -static void taosGetSystemTimezone() { - /* load time zone string from /etc/localtime */ - char buf[4096]; - char *tz = NULL; - { - int n = readlink("/etc/localtime", buf, sizeof(buf)); - if (n < 0) { - //printf("read /etc/localtime error, reason:%s", strerror(errno)); - return; - } - buf[n] = '\0'; - for (int i = n - 1; i >= 0; --i) { - if (buf[i] == '/') { - if (tz) { - tz = buf + i + 1; - break; - } - tz = buf + i + 1; - } - } - if (!tz || 0 == strchr(tz, '/')) { - //printf("parsing /etc/localtime failed"); - return; - } - - setenv("TZ", tz, 1); - tzset(); - } - - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], - -timezone / 3600); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - //printf("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * originally from src/os/src/detail/osSysinfo.c - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - /* if user does not specify the charset, extract it from locale */ - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} void taosKillSystem() { //printf("function taosKillSystem, exit!"); @@ -386,9 +264,6 @@ void taosGetSystemInfo() { long page_size = sysconf(_SC_PAGESIZE); tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); tsPageSize = page_size; - - taosGetSystemTimezone(); - taosGetSystemLocale(); } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { @@ -441,7 +316,7 @@ void taosSetCoreDump() {} int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { struct statvfs info; if (statvfs(dataDir, &info)) { - //printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + //printf("failed to get disk size, dataDir:%s errno:%s", osDataDir(), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } else { @@ -625,110 +500,6 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { return true; } -static void taosGetSystemTimezone() { - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* load time zone string from /etc/timezone */ - FILE *f = fopen("/etc/timezone", "r"); - char buf[68] = {0}; - if (f != NULL) { - int len = fread(buf, 64, 1, f); - if (len < 64 && ferror(f)) { - fclose(f); - //printf("read /etc/timezone error, reason:%s", strerror(errno)); - return; - } - - fclose(f); - - buf[sizeof(buf) - 1] = 0; - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } - - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - setenv("TZ", buf, 1); - } - } - // get and set default timezone - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; - tz += daylight; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - //printf("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - // if user does not specify the charset, extract it from locale - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } @@ -957,8 +728,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemTimezone(); - taosGetSystemLocale(); } void taosKillSystem() { diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c new file mode 100644 index 0000000000000000000000000000000000000000..a0ea01596fe5c811582073ec0809f66855d54f6e --- /dev/null +++ b/source/os/src/osTimezone.c @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { +#ifdef WINDOWS + char winStr[TD_LOCALE_LEN * 2]; + sprintf(winStr, "TZ=%s", inTimezone); + putenv(winStr); +#else + setenv("TZ", inTimezone, 1); +#endif + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char **tzname = _tzname; +#endif +#endif + + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + tz += daylight; + + /* + * format: + * (CST, +0800) + * (BST, +0100) + */ + + sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + *outDaylight = daylight; +} + +void taosGetSystemTimezone(char *outTimezone) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *tz = getenv("TZ"); + if (tz == NULL || strlen(tz) == 0) { + strcpy(outTimezone, "not configured"); + } else { + strcpy(outTimezone, tz); + } + +#elif defined(_TD_DARWIN_64) + char buf[4096] = {0}; + char *tz = NULL; + { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n < 0) { + printf("read /etc/localtime error, reason:%s", strerror(errno)); + return; + } + buf[n] = '\0'; + for (int i = n - 1; i >= 0; --i) { + if (buf[i] == '/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } + } + if (!tz || 0 == strchr(tz, '/')) { + printf("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); + } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + -timezone / 3600); + +#else + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* load time zone string from /etc/timezone */ + FILE *f = fopen("/etc/timezone", "r"); + char buf[68] = {0}; + if (f != NULL) { + int len = fread(buf, 64, 1, f); + if (len < 64 && ferror(f)) { + fclose(f); + // printf("read /etc/timezone error, reason:%s", strerror(errno)); + return; + } + + fclose(f); + + buf[sizeof(buf) - 1] = 0; + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } + + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } + } + // get and set default timezone + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ + int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + tz += daylight; + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + +#endif +} diff --git a/source/util/src/compare.c b/source/util/src/compare.c index 6124de945969e329310d316e6c6d728f446d02d9..1b1fa417546c611eb3b97e536b5cd8cfb569e8d1 100644 --- a/source/util/src/compare.c +++ b/source/util/src/compare.c @@ -466,7 +466,7 @@ int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -489,7 +489,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } } - if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -522,17 +522,17 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = compareStrPatternMatch; - } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ comparFn = compareStrPatternNotMatch; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_IN) { comparFn = compareChkInString; - } else if (optr == TSDB_RELATION_NOT_IN) { + } else if (optr == OP_TYPE_NOT_IN) { comparFn = compareChkNotInString; } else { /* normal relational comparFn */ comparFn = compareLenPrefixedStr; @@ -542,17 +542,17 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { + } else if (optr == OP_TYPE_LIKE) { comparFn = compareWStrPatternMatch; - } else if (optr == TSDB_RELATION_NOT_LIKE) { + } else if (optr == OP_TYPE_NOT_LIKE) { comparFn = compareWStrPatternNotMatch; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_IN) { comparFn = compareChkInString; - } else if (optr == TSDB_RELATION_NOT_IN) { + } else if (optr == OP_TYPE_NOT_IN) { comparFn = compareChkNotInString; } else { comparFn = compareLenPrefixedWStr; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c deleted file mode 100644 index 469da11d9312b6357711ac05777a0ea1d933b605..0000000000000000000000000000000000000000 --- a/source/util/src/tconfig.c +++ /dev/null @@ -1,507 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tconfig.h" -#include "tutil.h" -#include "ulog.h" - -SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; -int32_t tsGlobalConfigNum = 0; - -static char *tsGlobalUnit[] = { - " ", - "(%)", - "(GB)", - "(Mb)", - "(byte)", - "(s)", - "(ms)" -}; - -char *tsCfgStatusStr[] = { - "none", - "system default", - "config file", - "taos_options", - "program argument list" -}; - -static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) { - float value = (float)atof(input_value); - float *option = (float *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) { - double value = atof(input_value); - double *option = (double *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int32_t *option = (int32_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int16_t *option = (int16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - uint16_t *option = (uint16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (uint16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int8_t *option = (int8_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int8_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int)strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option, - input_value, cfg->ptrLength, option); - return false; - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - taosExpandDir(input_value, option, cfg->ptrLength); - taosRealPath(option, cfg->ptrLength); - - if (taosMkDir(option) != 0) { - uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, - strerror(errno)); - return false; - } - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } - - return true; -} - -static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) { - uint32_t value = taosInetAddr(input_value); - char * option = (char *)cfg->ptr; - if (value == INADDR_NONE) { - uError("config option:%s, input value:%s, is not a valid ip address, use default value:%s", - cfg->option, input_value, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } -} - -static void taosReadStringConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int) strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", - cfg->option, input_value, cfg->ptrLength, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } -} - -static void taosReadLogOption(char *option, char *value) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG) || !(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - // if (strcasecmp(cfg->option, "debugFlag") == 0) { - // taosSetAllDebugFlag(); - // } - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - default: - break; - } - break; - } -} - -SGlobalCfg *taosGetConfigOption(const char *option) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (strcasecmp(cfg->option, option) != 0) continue; - return cfg; - } - return NULL; -} - -static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - taosReadInt8Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT16: - taosReadInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - break; - case TAOS_CFG_VTYPE_UINT16: - taosReadUInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_FLOAT: - taosReadFloatConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DOUBLE: - taosReadDoubleConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_STRING: - taosReadStringConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_IPSTR: - taosReadIpStrConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DATA_DIRCTORY: - if (taosReadDirectoryConfig(cfg, value)) { - // taosReadDataDirCfg(value, value2, value3); - } - break; - default: - uError("config option:%s, input value:%s, can't be recognized", option, value); - break; - } - break; - } -} - -void taosAddConfigOption(SGlobalCfg cfg) { - tsGlobalConfig[tsGlobalConfigNum++] = cfg; -} - -void taosReadGlobalLogCfg() { - FILE * fp; - char * line, *option, *value; - int olen, vlen; - char fileName[PATH_MAX] = {0}; - - taosExpandDir(configDir, configDir, PATH_MAX); - taosReadLogOption("logDir", tsLogDir); - - sprintf(fileName, "%s/taos.cfg", configDir); - fp = fopen(fileName, "r"); - if (fp == NULL) { - printf("\nconfig file:%s not found, all variables are set to default\n", fileName); - return; - } - - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = NULL; - olen = vlen = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - taosReadLogOption(option, value); - } - - tfree(line); - fclose(fp); -} - -int32_t taosReadCfgFromFile() { - char * line, *option, *value, *value2, *value3; - int olen, vlen, vlen2, vlen3; - char fileName[PATH_MAX] = {0}; - - sprintf(fileName, "%s/taos.cfg", configDir); - FILE *fp = fopen(fileName, "r"); - if (fp == NULL) { - fp = fopen(configDir, "r"); - if (fp == NULL) { - return -1; - } - } - - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = value2 = value3 = NULL; - olen = vlen = vlen2 = vlen3 = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - paGetToken(value + vlen + 1, &value2, &vlen2); - if (vlen2 != 0) { - value2[vlen2] = 0; - paGetToken(value2 + vlen2 + 1, &value3, &vlen3); - if (vlen3 != 0) value3[vlen3] = 0; - } - - taosReadConfigOption(option, value, value2, value3); - } - - fclose(fp); - - tfree(line); - - // if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - // taosSetAllDebugFlag(); - // } - - return 0; -} - -void taosPrintCfg() { - uInfo(" taos config & system info:"); - uInfo("=================================="); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_DOUBLE: - uInfo(" %s:%s%f%s", cfg->option, blank, *((double *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - uInfo(" %s:%s%s%s", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } - } - - taosPrintOsInfo(); - uInfo("=================================="); -} - -static void taosDumpCfg(SGlobalCfg *cfg) { - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } -} - -void taosDumpGlobalCfg() { - printf("taos global config:\n"); - printf("==================================\n"); - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue; - - taosDumpCfg(cfg); - } - - printf("\ntaos local config:\n"); - printf("==================================\n"); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue; - - taosDumpCfg(cfg); - } -} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2b53a769ff5e4ce1afbb14e68383c77fbbb10c73..e6bef5f8b973d237aa4ef0f45f64e1a67702348f 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -81,6 +81,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") +TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 12aa77214f6109e9a33082c428a7bb7722a62835..ed55ef91b033442d667b9cea938e3abc6f504044 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "tlog.h" #include "os.h" -#include "tnote.h" #include "tutil.h" #include "ulog.h" @@ -70,29 +69,26 @@ typedef struct { int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; -int8_t tsAsyncLog = 1; -float tsTotalLogDirGB = 0; -float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 1.0f; +bool tsAsyncLog = true; +bool tsLogInited = false; int64_t asyncLogLostLines = 0; int32_t writeInterval = DEFAULT_LOG_INTERVAL; // log int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; +int32_t mDebugFlag = 131; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; -int32_t qDebugFlag = 131; -int32_t rpcDebugFlag = 131; +int32_t tmrDebugFlag = 131; int32_t uDebugFlag = 131; -int32_t debugFlag = 0; -int32_t sDebugFlag = 135; +int32_t rpcDebugFlag = 131; +int32_t qDebugFlag = 131; int32_t wDebugFlag = 135; +int32_t sDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t tqDebugFlag = 135; -int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; int64_t dbgEmptyW = 0; @@ -120,11 +116,18 @@ static int32_t taosStartLog() { return 0; } -int32_t taosInitLog(char *logName, int numOfLogLines, int maxFiles) { +int32_t taosInitLog(const char *logName, int maxFiles) { + if (tsLogInited) return 0; + osUpdate(); + + char fullName[PATH_MAX] = {0}; + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", osLogDir(), logName); + tsLogObj.logHandle = taosLogBuffNew(TSDB_DEFAULT_LOG_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; - if (taosOpenLogFile(logName, numOfLogLines, maxFiles) < 0) return -1; + if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; if (taosStartLog() < 0) return -1; + tsLogInited = true; return 0; } @@ -184,7 +187,7 @@ static void taosKeepOldLog(char *oldName) { } } - taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays)); + taosRemoveOldFiles(osLogDir(), TABS(tsLogKeepDays)); } static void *taosThreadToOpenNewFile(void *param) { @@ -217,7 +220,7 @@ static void *taosThreadToOpenNewFile(void *param) { uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintCfg(); + // taosPrintCfg(); taosKeepOldLog(keepName); return NULL; @@ -377,12 +380,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { } void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; va_list argpointer; char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; @@ -433,16 +431,10 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); - if (dflag == 255) nInfo(buffer, len); } void taosDumpData(unsigned char *msg, int32_t len) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; char temp[256]; int32_t i, pos = 0, c = 0; @@ -465,12 +457,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; va_list argpointer; char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE]; @@ -773,3 +760,23 @@ void taosPrintOsInfo() { uInfo(" os version: %s", info.version); uInfo(" os machine: %s", info.machine); } + +void taosSetAllDebugFlag(int32_t flag) { + if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; + + dDebugFlag = flag; + vDebugFlag = flag; + mDebugFlag = flag; + cDebugFlag = flag; + jniDebugFlag = flag; + uDebugFlag = flag; + rpcDebugFlag = flag; + qDebugFlag = flag; + wDebugFlag = flag; + sDebugFlag = flag; + tsdbDebugFlag = flag; + tqDebugFlag = flag; + fsDebugFlag = flag; + + uInfo("all debug flag are set to %d", flag); +} \ No newline at end of file diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c deleted file mode 100644 index d9356b7e4049cfdf9bc77ff1606e24993758bd6c..0000000000000000000000000000000000000000 --- a/source/util/src/tnote.c +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tutil.h" -#include "tdef.h" -#include "tnote.h" - -SNoteObj tsHttpNote; -SNoteObj tsTscNote; -SNoteObj tsInfoNote; - -static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote); -static void taosCloseNoteByFd(int32_t oldFd, SNoteObj *pNote); - -static void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, char *name) { - memset(pNote, 0, sizeof(SNoteObj)); - pNote->fileNum = 1; - pNote->fd = -1; - - if (taosOpenNoteWithMaxLines(name, numOfLines, maxNotes, pNote) < 0) { - fprintf(stderr, "failed to init note file\n"); - } - - taosNotePrint(pNote, "=================================================="); - taosNotePrint(pNote, "=================== new note ==================="); - taosNotePrint(pNote, "=================================================="); -} - -int32_t taosInitNotes() { - char name[TSDB_FILENAME_LEN * 2] = {0}; - -#if 0 - if (tsTscEnableRecordSql) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/tscsql-%d", tsLogDir, taosGetPId()); - taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); - } - -#endif - return 0; -} - -static bool taosLockNote(int32_t fd, SNoteObj *pNote) { - if (fd < 0) return false; - - if (pNote->fileNum > 1) { - int32_t ret = (int32_t)taosLockFile(fd); - if (ret == 0) { - return true; - } - } - - return false; -} - -static void taosUnLockNote(int32_t fd, SNoteObj *pNote) { - if (fd < 0) return; - - if (pNote->fileNum > 1) { - taosUnLockFile(fd); - } -} - -static void *taosThreadToOpenNewNote(void *param) { - char name[NOTE_FILE_NAME_LEN * 2]; - SNoteObj *pNote = (SNoteObj *)param; - - setThreadName("openNewNote"); - - pNote->flag ^= 1; - pNote->lines = 0; - sprintf(name, "%s.%d", pNote->name, pNote->flag); - - taosUmaskFile(0); - - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { - return NULL; - } - - taosLockNote(fd, pNote); - (void)taosLSeekFile(fd, 0, SEEK_SET); - - int32_t oldFd = pNote->fd; - pNote->fd = fd; - pNote->lines = 0; - pNote->openInProgress = 0; - taosNotePrint(pNote, "=============== new note is opened ============="); - - taosCloseNoteByFd(oldFd, pNote); - return NULL; -} - -static int32_t taosOpenNewNote(SNoteObj *pNote) { - pthread_mutex_lock(&pNote->mutex); - - if (pNote->lines > pNote->maxLines && pNote->openInProgress == 0) { - pNote->openInProgress = 1; - - taosNotePrint(pNote, "=============== open new note =================="); - pthread_t pattern; - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void *)pNote); - pthread_attr_destroy(&attr); - } - - pthread_mutex_unlock(&pNote->mutex); - - return pNote->fd; -} - -static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) { - int32_t fd = taosOpenFileCreateWrite(noteName); - if (fd < 0) { - fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); - return true; - } - - if (taosLockNote(fd, pNote)) { - taosUnLockNote(fd, pNote); - taosCloseFile(fd); - return false; - } else { - taosCloseFile(fd); - return true; - } -} - -static void taosGetNoteName(char *fn, SNoteObj *pNote) { - if (pNote->fileNum > 1) { - for (int32_t i = 0; i < pNote->fileNum; i++) { - char fileName[NOTE_FILE_NAME_LEN]; - - sprintf(fileName, "%s%d.0", fn, i); - bool file1open = taosCheckNoteIsOpen(fileName, pNote); - - sprintf(fileName, "%s%d.1", fn, i); - bool file2open = taosCheckNoteIsOpen(fileName, pNote); - - if (!file1open && !file2open) { - sprintf(pNote->name, "%s%d", fn, i); - return; - } - } - } - - if (strlen(fn) < NOTE_FILE_NAME_LEN) { - strcpy(pNote->name, fn); - } -} - -static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) { - char name[NOTE_FILE_NAME_LEN * 2] = {0}; - int32_t size; - int32_t logstat0_mtime, logstat1_mtime; - - pNote->maxLines = maxLines; - pNote->fileNum = maxNoteNum; - taosGetNoteName(fn, pNote); - - if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".0"); - } - bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0; - - if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".1"); - } - bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0; - - if (!log0Exist && !log1Exist) { - pNote->flag = 0; - } else if (!log1Exist) { - pNote->flag = 0; - } else if (!log0Exist) { - pNote->flag = 1; - } else { - pNote->flag = (logstat0_mtime > logstat1_mtime) ? 0 : 1; - } - - char noteName[NOTE_FILE_NAME_LEN * 2] = {0}; - sprintf(noteName, "%s.%d", pNote->name, pNote->flag); - pthread_mutex_init(&pNote->mutex, NULL); - - taosUmaskFile(0); - pNote->fd = taosOpenFileCreateWrite(noteName); - - if (pNote->fd < 0) { - fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - taosLockNote(pNote->fd, pNote); - - // only an estimate for number of lines - int64_t filestat_size; - if (taosFStatFile(pNote->fd, &filestat_size, NULL) < 0) { - fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - size = (int32_t)filestat_size; - pNote->lines = size / 60; - - taosLSeekFile(pNote->fd, 0, SEEK_END); - - return 0; -} - -void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) { - if (pNote->fd <= 0) return; - taosWriteFile(pNote->fd, buffer, len); - - if (pNote->maxLines > 0) { - pNote->lines++; - if ((pNote->lines > pNote->maxLines) && (pNote->openInProgress == 0)) taosOpenNewNote(pNote); - } -} - -void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { - va_list argpointer; - char buffer[MAX_NOTE_LINE_SIZE + 2]; - int32_t len; - struct tm Tm, *ptm; - struct timeval timeSecs; - time_t curTime; - - taosGetTimeOfDay(&timeSecs); - curTime = timeSecs.tv_sec; - ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); - va_start(argpointer, format); - len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); - va_end(argpointer); - - if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2; - - buffer[len++] = '\n'; - buffer[len] = 0; - - taosNotePrintBuffer(pNote, buffer, len); -} - -// static void taosCloseNote(SNoteObj *pNote) { taosCloseNoteByFd(pNote->fd, pNote); } - -static void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) { - if (fd >= 0) { - taosUnLockNote(fd, pNote); - taosCloseFile(fd); - } -} diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 65101a5e07577e29c4678bec5dba1ac3a38a7347..ca0c6e9a2443892b8813b106f72a0eb39e87cac9 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -111,7 +111,6 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -int32_t tmrDebugFlag = 131; uint32_t tsMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 54aa6a5f346b481e387593fcbd40077ca1a979ec..1484ed2182b35aba1a9af093a1ceb0840b90736e 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("tref.log", 5000000, 10); + taosInitLog("tref.log", 10); SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads); diff --git a/tests b/tests index 4da4bb00c5615f0b1bea04340abef5d9d15a96b4..12233db374f1fe97b327e89a3442c631578ad38d 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 4da4bb00c5615f0b1bea04340abef5d9d15a96b4 +Subproject commit 12233db374f1fe97b327e89a3442c631578ad38d diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4186f0dae1038f2336565a99b03953b0e99a1e8d..33774bdd058b516189d7c0552c08622e10ac8a8d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -68,11 +68,11 @@ TAOS *shellInit(SShellArguments *_args) { printf("\n"); if (!_args->is_use_passwd) { #ifdef TD_WINDOWS - strcpy(tsOsName, "Windows"); + strcpy(osName(), "Windows"); #elif defined(TD_DARWIN) - strcpy(tsOsName, "Darwin"); + strcpy(osName(), "Darwin"); #endif - printf(CLIENT_VERSION, tsOsName, taos_get_client_info()); + printf(CLIENT_VERSION, osName(), taos_get_client_info()); } fflush(stdout); diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 81d8c0a35bf86acc47c5f040c18ac97a6d97d53c..caba986ce7f5326a3ec15ebc64220cd50dc67581 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -20,6 +20,7 @@ #include "shellCommand.h" #include "tkey.h" #include "ulog.h" +#include "version.h" #include #include @@ -183,8 +184,7 @@ static void parse_args( for (int i = 1; i < argc; i++) { if ((strncmp(argv[i], "-p", 2) == 0) || (strncmp(argv[i], "--password", 10) == 0)) { - strcpy(tsOsName, "Linux"); - printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); + printf(LINUXCLIENT_VERSION, osName(), taos_get_client_info()); if ((strlen(argv[i]) == 2) || (strncmp(argv[i], "--password", 10) == 0)) { printf("Enter password: "); diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 607dc792572ca07db43942d89b98f6d97e358eae..8a1763c4fc8d2f778bf6c0b2874ff16c5868cf48 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -15,7 +15,6 @@ #include "os.h" #include "shell.h" -#include "tconfig.h" #include "tglobal.h" pthread_t pid;