Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1193
Star 22018
Fork 4786
T
TDengine
提交 ae2d77e9 编写于 作者: S Shengliang Guan
浏览文件
操作

Merge remote-tracking branch 'origin/3.0' into feature/dnode3

上级 de8b6e73 1b1a6178
显示空白变更内容
内联 并排
Showing with 4454 addition and 3886 deletion
.devcontainer/devcontainer.json
浏览文件 @ ae2d77e9
......@@ -26,7 +26,8 @@
"eamodio.gitlens",
"matepek.vscode-catch2-test-adapter",
"spmeesseman.vscode-taskexplorer",
"cschlosser.doxdocgen"
"cschlosser.doxdocgen",
"urosvujosevic.explorer-manager"
],
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],
......
cmake/libuv_CMakeLists.txt.in
浏览文件 @ ae2d77e9
......@@ -4,7 +4,7 @@ ExternalProject_Add(libuv
GIT_REPOSITORY https://github.com/libuv/libuv.git
GIT_TAG v1.42.0
SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv"
BINARY_DIR ""
BINARY_DIR "${CMAKE_CONTRIB_DIR}/libuv"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
......
include/client/taos.h
浏览文件 @ ae2d77e9
......@@ -193,8 +193,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList);
DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision);
DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen);
DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen);
#ifdef __cplusplus
}
......
include/common/tmsg.h
浏览文件 @ ae2d77e9
......@@ -68,6 +68,14 @@ typedef uint16_t tmsg_t;
#define TSDB_IE_TYPE_DNODE_EXT 6
#define TSDB_IE_TYPE_DNODE_STATE 7
typedef enum {
HEARTBEAT_TYPE_MQ = 0,
HEARTBEAT_TYPE_QUERY = 1,
// types can be added here
//
HEARTBEAT_TYPE_MAX
} EHbType;
typedef enum _mgmt_table {
TSDB_MGMT_TABLE_START,
TSDB_MGMT_TABLE_ACCT,
......@@ -147,7 +155,7 @@ typedef struct {
typedef struct {
SClientHbKey connKey;
SHashObj* info; // hash<Slv.key, Sklv>
SHashObj* info; // hash<Skv.key, Skv>
} SClientHbReq;
typedef struct {
......@@ -173,7 +181,10 @@ static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) {
}
int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq);
void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq);
void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq);
int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp);
void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp);
static FORCE_INLINE void tFreeClientHbReq(void *pReq) {
SClientHbReq* req = (SClientHbReq*)pReq;
......@@ -182,14 +193,17 @@ static FORCE_INLINE void tFreeClientHbReq(void *pReq) {
}
int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq);
void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq);
void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq);
static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) {
SClientHbBatchReq *req = (SClientHbBatchReq*)pReq;
taosArrayDestroyEx(req->reqs, tFreeClientHbReq);
//taosArrayDestroyEx(req->reqs, tFreeClientHbReq);
free(pReq);
}
int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp);
void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp);
static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pKv->keyLen);
......@@ -220,6 +234,7 @@ static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey)
return buf;
}
typedef struct {
int32_t vgId;
char* dbName;
......@@ -359,6 +374,31 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) {
return buf;
}
typedef struct SMqHbRsp {
int8_t status; //idle or not
int8_t vnodeChanged;
int8_t epChanged; // should use new epset
int8_t reserved;
SEpSet epSet;
} SMqHbRsp;
static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) {
int tlen = 0;
tlen += taosEncodeFixedI8(buf, pRsp->status);
tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged);
tlen += taosEncodeFixedI8(buf, pRsp->epChanged);
tlen += taosEncodeSEpSet(buf, &pRsp->epSet);
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) {
buf = taosDecodeFixedI8(buf, &pRsp->status);
buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged);
buf = taosDecodeFixedI8(buf, &pRsp->epChanged);
buf = taosDecodeSEpSet(buf, &pRsp->epSet);
return buf;
}
typedef struct {
int32_t acctId;
int64_t clusterId;
......@@ -993,6 +1033,13 @@ typedef struct {
uint64_t taskId;
} SSinkDataReq;
typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
} SQueryContinueReq;
typedef struct {
SMsgHead header;
......@@ -1053,6 +1100,7 @@ typedef struct {
typedef struct {
int8_t igExists;
char* name;
char* sql;
char* physicalPlan;
char* logicalPlan;
} SCMCreateTopicReq;
......@@ -1061,6 +1109,7 @@ static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateT
int tlen = 0;
tlen += taosEncodeFixedI8(buf, pReq->igExists);
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeString(buf, pReq->sql);
tlen += taosEncodeString(buf, pReq->physicalPlan);
tlen += taosEncodeString(buf, pReq->logicalPlan);
return tlen;
......@@ -1069,6 +1118,7 @@ static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateT
static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) {
buf = taosDecodeFixedI8(buf, &(pReq->igExists));
buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeString(buf, &(pReq->sql));
buf = taosDecodeString(buf, &(pReq->physicalPlan));
buf = taosDecodeString(buf, &(pReq->logicalPlan));
return buf;
......@@ -1191,7 +1241,7 @@ typedef struct {
} SMVSubscribeRsp;
typedef struct {
char name[TSDB_TOPIC_FNAME_LEN];
char name[TSDB_TOPIC_NAME_LEN];
int8_t igExists;
int32_t execLen;
void* executor;
......
include/common/tmsgdef.h
浏览文件 @ ae2d77e9
......@@ -129,7 +129,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp)
TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL)
......@@ -170,6 +170,8 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp)
TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp)
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SCHEDULE_DATA_SINK, "vnode-schedule-data-sink", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
......
include/common/tname.h
浏览文件 @ ae2d77e9
......@@ -25,14 +25,12 @@
#define T_NAME_ACCT 0x1u
#define T_NAME_DB 0x2u
#define T_NAME_TABLE 0x4u
#define T_NAME_TOPIC 0x8u
typedef struct SName {
uint8_t type; //db_name_t, table_name_t
int32_t acctId;
char dbname[TSDB_DB_NAME_LEN];
char tname[TSDB_TABLE_NAME_LEN];
char topicName[TSDB_TOPIC_NAME_LEN];
} SName;
int32_t tNameExtractFullName(const SName* name, char* dst);
......
include/libs/parser/parsenodes.h
浏览文件 @ ae2d77e9
......@@ -43,15 +43,6 @@ typedef struct SField {
int32_t bytes;
} SField;
typedef struct SParseBasicCtx {
uint64_t requestId;
int32_t acctId;
const char *db;
void *pTransporter;
SEpSet mgmtEpSet;
struct SCatalog *pCatalog;
} SParseBasicCtx;
typedef struct SFieldInfo {
int16_t numOfOutput; // number of column in result
SField *final;
......
include/libs/parser/parser.h
浏览文件 @ ae2d77e9
......@@ -23,11 +23,17 @@ extern "C" {
#include "parsenodes.h"
typedef struct SParseContext {
SParseBasicCtx ctx;
uint64_t requestId;
int32_t acctId;
const char *db;
void *pTransporter;
SEpSet mgmtEpSet;
const char *pSql; // sql string
size_t sqlLen; // length of the sql string
char *pMsg; // extended error message if exists to help identifying the problem in sql statement.
int32_t msgLen; // max length of the msg
struct SCatalog *pCatalog;
} SParseContext;
/**
......@@ -38,7 +44,7 @@ typedef struct SParseContext {
* @param msg extended error message if exists.
* @return error code
*/
int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery);
int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQueryNode);
/**
* Return true if it is a ddl/dcl sql statement
......
include/libs/planner/planner.h
浏览文件 @ ae2d77e9
......@@ -93,6 +93,9 @@ typedef struct SScanPhyNode {
SPhyNode node;
uint64_t uid; // unique id of the table
int8_t tableType;
int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
int32_t count; // repeat count
int32_t reverse; // reverse scan count
} SScanPhyNode;
typedef SScanPhyNode SSystemTableScanPhyNode;
......@@ -117,6 +120,25 @@ typedef struct SExchangePhyNode {
SArray *pSrcEndPoints; // SEpAddr, scheduler fill by calling qSetSuplanExecutionNode
} SExchangePhyNode;
typedef enum EAggAlgo {
AGG_ALGO_PLAIN = 1, // simple agg across all input rows
AGG_ALGO_SORTED, // grouped agg, input must be sorted
AGG_ALGO_HASHED // grouped agg, use internal hashtable
} EAggAlgo;
typedef enum EAggSplit {
AGG_SPLIT_PRE = 1, // first level agg, maybe don't need calculate the final result
AGG_SPLIT_FINAL // second level agg, must calculate the final result
} EAggSplit;
typedef struct SAggPhyNode {
SPhyNode node;
EAggAlgo aggAlgo; // algorithm used by agg operator
EAggSplit aggSplit; // distributed splitting mode
SArray *pExprs; // SExprInfo list, these are expression list of group_by_clause and parameter expression of aggregate function
SArray *pGroupByList; // SColIndex list, but these must be column node
} SAggPhyNode;
typedef struct SSubplanId {
uint64_t queryId;
uint64_t templateId;
......
include/libs/planner/plannerOp.h
浏览文件 @ ae2d77e9
......@@ -30,7 +30,7 @@ OP_ENUM_MACRO(TagScan)
OP_ENUM_MACRO(SystemTableScan)
OP_ENUM_MACRO(Aggregate)
OP_ENUM_MACRO(Project)
OP_ENUM_MACRO(Groupby)
// OP_ENUM_MACRO(Groupby)
OP_ENUM_MACRO(Limit)
OP_ENUM_MACRO(SLimit)
OP_ENUM_MACRO(TimeWindow)
......
include/libs/qworker/qworker.h
浏览文件 @ ae2d77e9
......@@ -22,9 +22,18 @@ extern "C" {
#include "trpc.h"
enum {
NODE_TYPE_VNODE = 1,
NODE_TYPE_QNODE,
NODE_TYPE_SNODE,
};
typedef struct SQWorkerCfg {
uint32_t maxSchedulerNum;
uint32_t maxResCacheNum;
uint32_t maxTaskNum;
uint32_t maxSchTaskNum;
} SQWorkerCfg;
......@@ -39,11 +48,17 @@ typedef struct {
uint64_t numOfErrors;
} SQWorkerStat;
typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *);
int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt);
int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp);
int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
......
include/libs/tkv/tkv.h 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TKV_H_
#define _TD_TKV_H_
#if 0
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// Types exported
typedef struct STkvDb STkvDb;
typedef struct STkvOpts STkvOpts;
typedef struct STkvCache STkvCache;
typedef struct STkvReadOpts STkvReadOpts;
typedef struct STkvWriteOpts STkvWriteOpts;
// DB operations
STkvDb *tkvOpen(const STkvOpts *options, const char *path);
void tkvClose(STkvDb *db);
void tkvPut(STkvDb *db, const STkvWriteOpts *, const char *key, size_t keylen, const char *val, size_t vallen);
char * tkvGet(STkvDb *db, const STkvReadOpts *, const char *key, size_t keylen, size_t *vallen);
void tkvCommit(STkvDb *db);
// DB options
STkvOpts *tkvOptsCreate();
void tkvOptsDestroy(STkvOpts *);
void tkvOptionsSetCache(STkvOpts *, STkvCache *);
void tkvOptsSetCreateIfMissing(STkvOpts *, unsigned char);
// DB cache
typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType;
STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type);
void tkvCacheDestroy(STkvCache *);
// STkvReadOpts
STkvReadOpts *tkvReadOptsCreate();
void tkvReadOptsDestroy(STkvReadOpts *);
// STkvWriteOpts
STkvWriteOpts *tkvWriteOptsCreate();
void tkvWriteOptsDestroy(STkvWriteOpts *);
#ifdef __cplusplus
}
#endif
#endif
#endif /*_TD_TKV_H_*/
\ No newline at end of file
include/util/taoserror.h
浏览文件 @ ae2d77e9
......@@ -307,6 +307,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) //"Database write operation denied")
#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) //"Database is syncing")
#define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) //"Invalid tsdb state")
#define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) // "Table not exists")
// tsdb
#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) //"Invalid table ID")
......
include/util/tarray.h
浏览文件 @ ae2d77e9
......@@ -23,6 +23,19 @@ extern "C" {
#include "os.h"
#include "talgo.h"
#if 0
#define TARRAY(TYPE) \
struct { \
int32_t tarray_size_; \
int32_t tarray_neles_; \
struct TYPE* td_array_data_; \
}
#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_
#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_
#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx)
#endif
#define TARRAY_MIN_SIZE 8
#define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize))
#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize)
......@@ -57,7 +70,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize);
* @param nEles
* @return
*/
void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles);
void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles);
/**
*
......@@ -65,7 +78,7 @@ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles);
* @param pData position array list
* @param numOfElems the number of removed position
*/
void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems);
void taosArrayRemoveBatch(SArray* pArray, const int32_t* pData, int32_t numOfElems);
/**
*
......@@ -73,7 +86,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle
* @param comparFn
* @param fp
*/
void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*));
void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*));
/**
* add all element from the source array list into the destination
......@@ -242,7 +255,6 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t
*/
char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags);
/**
* sort the pointer data in the array
* @param pArray
......@@ -251,7 +263,7 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t
* @return
*/
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param);
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
#ifdef __cplusplus
}
......
include/util/tcoding.h
浏览文件 @ ae2d77e9
......@@ -351,6 +351,7 @@ static FORCE_INLINE void *taosDecodeString(void *buf, char **value) {
buf = taosDecodeVariantU64(buf, &size);
*value = (char *)malloc((size_t)size + 1);
if (*value == NULL) return NULL;
memcpy(*value, buf, (size_t)size);
......
include/util/tdef.h
浏览文件 @ ae2d77e9
......@@ -181,7 +181,7 @@ do { \
#define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE
#define TSDB_MAX_SQL_SHOW_LEN 512
#define TSDB_MAX_SQL_SHOW_LEN 1024
#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb
#define TSDB_APP_NAME_LEN TSDB_UNI_LEN
......
source/client/inc/clientHb.h 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tarray.h"
#include "thash.h"
#include "tmsg.h"
#define HEARTBEAT_INTERVAL 1500 // ms
typedef enum {
HEARTBEAT_TYPE_MQ = 0,
// types can be added here
//
HEARTBEAT_TYPE_MAX
} EHbType;
typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq);
typedef struct SAppHbMgr {
// statistics
int32_t reportCnt;
int32_t connKeyCnt;
int64_t reportBytes; // not implemented
int64_t startTime;
// ctl
SRWLatch lock; // lock is used in serialization
// connection
void* transporter;
SEpSet epSet;
// info
SHashObj* activeInfo; // hash<SClientHbKey, SClientHbReq>
SHashObj* getInfoFuncs; // hash<SClientHbKey, FGetConnInfo>
} SAppHbMgr;
typedef struct SClientHbMgr {
int8_t inited;
// ctl
int8_t threadStop;
pthread_t thread;
pthread_mutex_t lock; // used when app init and cleanup
SArray* appHbMgrs; // SArray<SAppHbMgr*> one for each cluster
FHbRspHandle handle[HEARTBEAT_TYPE_MAX];
} SClientHbMgr;
// TODO: embed param into function
// return type: SArray<Skv>
typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param);
// global, called by mgmt
int hbMgrInit();
void hbMgrCleanUp();
int hbHandleRsp(SClientHbBatchRsp* hbRsp);
// cluster level
SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet);
void appHbMgrCleanup(SAppHbMgr* pAppHbMgr);
// conn level
int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func);
void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen);
// mq
void hbMgrInitMqHbRspHandle();
source/client/inc/clientInt.h
浏览文件 @ ae2d77e9
......@@ -31,6 +31,41 @@ extern "C" {
#include "trpc.h"
#include "query.h"
#define HEARTBEAT_INTERVAL 1500 // ms
typedef struct SAppInstInfo SAppInstInfo;
typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq);
typedef struct SAppHbMgr {
// statistics
int32_t reportCnt;
int32_t connKeyCnt;
int64_t reportBytes; // not implemented
int64_t startTime;
// ctl
SRWLatch lock; // lock is used in serialization
// connection
SAppInstInfo* pAppInstInfo;
// info
SHashObj* activeInfo; // hash<SClientHbKey, SClientHbReq>
SHashObj* getInfoFuncs; // hash<SClientHbKey, FGetConnInfo>
} SAppHbMgr;
typedef struct SClientHbMgr {
int8_t inited;
// ctl
int8_t threadStop;
pthread_t thread;
pthread_mutex_t lock; // used when app init and cleanup
SArray* appHbMgrs; // SArray<SAppHbMgr*> one for each cluster
FHbRspHandle handle[HEARTBEAT_TYPE_MAX];
} SClientHbMgr;
// TODO: embed param into function
// return type: SArray<Skv>
typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param);
typedef struct SQueryExecMetric {
int64_t start; // start timestamp
int64_t parsed; // start to parse
......@@ -55,15 +90,15 @@ typedef struct SHeartBeatInfo {
void *pTimer; // timer, used to send request msg to mnode
} SHeartBeatInfo;
typedef struct SAppInstInfo {
struct SAppInstInfo {
int64_t numOfConns;
SCorEpSet mgmtEp;
SInstanceSummary summary;
SList *pConnList; // STscObj linked list
int64_t clusterId;
void *pTransporter;
SHeartBeatInfo hb;
} SAppInstInfo;
struct SAppHbMgr *pAppHbMgr;
};
typedef struct SAppInfo {
int64_t startTime;
......@@ -81,25 +116,28 @@ typedef struct STscObj {
char db[TSDB_DB_FNAME_LEN];
int32_t acctId;
uint32_t connId;
int32_t connType;
uint64_t id; // ref ID returned by taosAddRef
void *pTransporter;
pthread_mutex_t mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj from this tscObj
int32_t numOfReqs; // number of sqlObj bound to this connection
SAppInstInfo *pAppInfo;
} STscObj;
typedef struct SMqConsumer {
STscObj* pTscObj;
} SMqConsumer;
typedef struct SReqResultInfo {
const char *pRspMsg;
const char *pData;
TAOS_FIELD *fields;
uint32_t numOfCols;
int32_t *length;
TAOS_ROW row;
char **pCol;
uint32_t numOfRows;
uint32_t current;
bool completed;
} SReqResultInfo;
typedef struct SShowReqInfo {
......@@ -113,8 +151,9 @@ typedef struct SRequestSendRecvBody {
tsem_t rspSem; // not used now
void* fp;
SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed.
struct SSchJob *pQueryJob; // query job, created according to sql query DAG.
SDataBuf requestMsg;
struct SSchJob *pQueryJob; // query job, created according to sql query DAG.
struct SQueryDag *pDag; // the query dag, generated according to the sql statement.
SReqResultInfo resInfo;
} SRequestSendRecvBody;
......@@ -130,7 +169,6 @@ typedef struct SRequestObj {
char *msgBuf;
void *pInfo; // sql parse info, generated by parser module
int32_t code;
uint64_t affectedRows; // todo remove it
SQueryExecMetric metric;
SRequestSendRecvBody body;
} SRequestObj;
......@@ -171,6 +209,26 @@ void *doFetchRow(SRequestObj* pRequest);
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
// --- heartbeat
// global, called by mgmt
int hbMgrInit();
void hbMgrCleanUp();
int hbHandleRsp(SClientHbBatchRsp* hbRsp);
// cluster level
SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo);
void appHbMgrCleanup(SAppHbMgr* pAppHbMgr);
// conn level
int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func);
void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen);
// --- mq
void hbMgrInitMqHbRspHandle();
#ifdef __cplusplus
}
#endif
......
source/client/src/clientEnv.c
浏览文件 @ ae2d77e9
......@@ -89,13 +89,12 @@ static void tscInitLogFile() {
// todo close the transporter properly
void closeTransporter(STscObj* pTscObj) {
if (pTscObj == NULL || pTscObj->pTransporter == NULL) {
if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) {
return;
}
tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id);
rpcClose(pTscObj->pTransporter);
pTscObj->pTransporter = NULL;
tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id);
rpcClose(pTscObj->pAppInfo->pTransporter);
}
// TODO refactor
......@@ -140,10 +139,6 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI
}
pObj->pAppInfo = pAppInfo;
if (pAppInfo != NULL) {
pObj->pTransporter = pAppInfo->pTransporter;
}
tstrncpy(pObj->user, user, sizeof(pObj->user));
memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN);
......@@ -199,6 +194,7 @@ static void doDestroyRequest(void* p) {
tfree(pRequest->pInfo);
doFreeReqResultInfo(&pRequest->body.resInfo);
qDestroyQueryDag(pRequest->body.pDag);
deregisterRequest(pRequest);
tfree(pRequest);
......
source/client/src/clientHb.c
浏览文件 @ ae2d77e9
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "clientHb.h"
#include "clientInt.h"
#include "trpc.h"
static SClientHbMgr clientHbMgr = {0};
......@@ -21,10 +21,18 @@ static SClientHbMgr clientHbMgr = {0};
static int32_t hbCreateThread();
static void hbStopThread();
static int32_t hbMqHbRspHandle(SClientHbRsp* pReq) {
static int32_t hbMqHbRspHandle(SClientHbRsp* pRsp) {
return 0;
}
static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code) {
if (code != 0) {
return -1;
}
SClientHbRsp* pRsp = (SClientHbRsp*) pMsg->pData;
return hbMqHbRspHandle(pRsp);
}
void hbMgrInitMqHbRspHandle() {
clientHbMgr.handle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle;
}
......@@ -35,18 +43,18 @@ static FORCE_INLINE void hbMgrInitHandle() {
}
SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
SClientHbBatchReq* pReq = malloc(sizeof(SClientHbBatchReq));
if (pReq == NULL) {
SClientHbBatchReq* pBatchReq = malloc(sizeof(SClientHbBatchReq));
if (pBatchReq == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return NULL;
}
int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt);
pReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq));
pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq));
void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL);
while (pIter != NULL) {
taosArrayPush(pReq->reqs, pIter);
SClientHbReq* pOneReq = pIter;
taosArrayPush(pBatchReq->reqs, pOneReq);
taosHashClear(pOneReq->info);
pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
......@@ -59,10 +67,10 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
taosHashCopyKey(pIter, &connKey);
getConnInfoFp(connKey, NULL);
pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, pIter);
}
return pReq;
return pBatchReq;
}
static void* hbThreadFunc(void* param) {
......@@ -75,20 +83,48 @@ static void* hbThreadFunc(void* param) {
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
for(int i = 0; i < sz; i++) {
SAppHbMgr* pAppHbMgr = taosArrayGet(clientHbMgr.appHbMgrs, i);
SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr);
void* reqStr = NULL;
int tlen = tSerializeSClientHbBatchReq(&reqStr, pReq);
SMsgSendInfo info;
/*info.fp = hbHandleRsp;*/
SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt);
if (connCnt == 0) {
continue;
}
SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr);
if (pReq == NULL) {
continue;
}
int tlen = tSerializeSClientHbBatchReq(NULL, pReq);
void *buf = malloc(tlen);
if (buf == NULL) {
//TODO: error handling
break;
}
void *bufCopy = buf;
tSerializeSClientHbBatchReq(&bufCopy, pReq);
SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo));
if (pInfo == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
tFreeClientHbBatchReq(pReq);
free(buf);
break;
}
pInfo->fp = hbMqAsyncCallBack;
pInfo->msgInfo.pData = buf;
pInfo->msgInfo.len = tlen;
pInfo->msgType = TDMT_MND_HEARTBEAT;
pInfo->param = NULL;
pInfo->requestId = generateRequestId();
pInfo->requestObjRefId = 0;
SAppInstInfo *pAppInstInfo = pAppHbMgr->pAppInstInfo;
int64_t transporterId = 0;
asyncSendMsgToServer(pAppHbMgr->transporter, &pAppHbMgr->epSet, &transporterId, &info);
SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo);
tFreeClientHbBatchReq(pReq);
atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1);
taosMsleep(HEARTBEAT_INTERVAL);
}
taosMsleep(HEARTBEAT_INTERVAL);
}
return NULL;
}
......@@ -110,7 +146,8 @@ static void hbStopThread() {
atomic_store_8(&clientHbMgr.threadStop, 1);
}
SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) {
SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) {
hbMgrInit();
SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr));
if (pAppHbMgr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
......@@ -119,16 +156,27 @@ SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) {
// init stat
pAppHbMgr->startTime = taosGetTimestampMs();
// init connection info
pAppHbMgr->transporter = transporter;
pAppHbMgr->epSet = epSet;
// init app info
pAppHbMgr->pAppInstInfo = pAppInstInfo;
// init hash info
pAppHbMgr->activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
if (pAppHbMgr->activeInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
free(pAppHbMgr);
return NULL;
}
pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq;
// init getInfoFunc
pAppHbMgr->getInfoFuncs = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
if (pAppHbMgr->getInfoFuncs == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
free(pAppHbMgr);
return NULL;
}
taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr);
return pAppHbMgr;
}
......@@ -138,7 +186,7 @@ void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) {
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
for (int i = 0; i < sz; i++) {
SAppHbMgr* pTarget = taosArrayGet(clientHbMgr.appHbMgrs, i);
SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
if (pAppHbMgr == pTarget) {
taosHashCleanup(pTarget->activeInfo);
taosHashCleanup(pTarget->getInfoFuncs);
......@@ -171,7 +219,6 @@ void hbMgrCleanUp() {
if (old == 0) return;
taosArrayDestroy(clientHbMgr.appHbMgrs);
}
int hbHandleRsp(SClientHbBatchRsp* hbRsp) {
......
source/client/src/clientImpl.c
浏览文件 @ ae2d77e9
......@@ -13,20 +13,21 @@
#include "tpagedfile.h"
#include "tref.h"
#define CHECK_CODE_GOTO(expr, lable) \
#define CHECK_CODE_GOTO(expr, label) \
do { \
int32_t code = expr; \
if (TSDB_CODE_SUCCESS != code) { \
terrno = code; \
goto lable; \
goto label; \
} \
} while (0)
static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest);
static void destroySendMsgInfo(SMsgSendInfo* pMsgBody);
static void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
static bool stringLengthCheck(const char* str, size_t maxsize) {
static bool stringLengthCheck(const char* str, size_t maxsize) {
if (str == NULL) {
return false;
}
......@@ -57,7 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
return strdup(key);
}
static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo);
static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo);
static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema);
TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) {
......@@ -70,18 +71,18 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
return NULL;
}
char tmp[TSDB_DB_NAME_LEN] = {0};
char localDb[TSDB_DB_NAME_LEN] = {0};
if (db != NULL) {
if(!validateDbName(db)) {
terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
return NULL;
}
tstrncpy(tmp, db, sizeof(tmp));
strdequote(tmp);
tstrncpy(localDb, db, sizeof(localDb));
strdequote(localDb);
}
char secretEncrypt[32] = {0};
char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
if (auth == NULL) {
if (!validatePassword(pass)) {
terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH;
......@@ -110,18 +111,20 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
char* key = getClusterKey(user, secretEncrypt, ip, port);
// TODO: race condition here.
SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
if (pInst == NULL) {
SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo));
p->mgmtEp = epSet;
p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
p->pAppHbMgr = appHbMgrInit(p);
taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
pInst = &p;
}
tfree(key);
return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, *pInst);
return taosConnectImpl(user, &secretEncrypt[0], localDb, port, NULL, NULL, *pInst);
}
int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest) {
......@@ -150,23 +153,26 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) {
STscObj* pTscObj = pRequest->pTscObj;
SParseContext cxt = {
.ctx = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj), .pTransporter = pTscObj->pTransporter},
.requestId = pRequest->requestId,
.acctId = pTscObj->acctId,
.db = getConnectionDB(pTscObj),
.pSql = pRequest->sqlstr,
.sqlLen = pRequest->sqlLen,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.pTransporter = pTscObj->pAppInfo->pTransporter,
};
cxt.ctx.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.ctx.pCatalog);
cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tfree(cxt.ctx.db);
tfree(cxt.db);
return code;
}
code = qParseQuerySql(&cxt, pQuery);
tfree(cxt.ctx.db);
tfree(cxt.db);
return code;
}
......@@ -187,10 +193,10 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
pShowReqInfo->pArray = pDcl->pExtension;
}
}
asyncSendMsgToServer(pTscObj->pTransporter, &pDcl->epSet, &transporterId, pSendMsg);
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg);
} else {
SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, pSendMsg);
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, pEpSet, &transporterId, pSendMsg);
}
tsem_wait(&pRequest->body.rspSem);
......@@ -212,6 +218,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag)
SSubplan* pPlan = taosArrayGetP(pa, 0);
SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema);
setResSchemaInfo(pResInfo, pDataBlockSchema);
pRequest->type = TDMT_VND_QUERY;
}
......@@ -228,7 +235,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlo
SSchema* pSchema = &pDataBlockSchema->pSchema[i];
pResInfo->fields[i].bytes = pSchema->bytes;
pResInfo->fields[i].type = pSchema->type;
tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name));
tstrncpy(pResInfo->fields[i].name, pSchema->name, tListLen(pResInfo->fields[i].name));
}
}
......@@ -236,7 +243,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) {
if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res);
int32_t code = scheduleExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res);
if (code != TSDB_CODE_SUCCESS) {
// handle error and retry
} else {
......@@ -245,43 +252,165 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) {
}
}
pRequest->affectedRows = res.numOfRows;
return res.code;
pRequest->body.resInfo.numOfRows = res.numOfRows;
pRequest->code = res.code;
return pRequest->code;
}
return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob);
}
typedef struct tmq_t tmq_t;
typedef struct SMqClientTopic {
// subscribe info
int32_t sqlLen;
char* sql;
char* topicName;
int64_t topicId;
// statistics
int64_t consumeCnt;
// offset
int64_t committedOffset;
int64_t currentOffset;
//connection info
int32_t vgId;
SEpSet epSet;
} SMqClientTopic;
typedef struct tmq_resp_err_t {
int32_t code;
} tmq_resp_err_t;
typedef struct tmq_topic_vgroup_list_t {
char* topicName;
int32_t vgId;
int64_t committedOffset;
} tmq_topic_vgroup_list_t;
typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param));
typedef struct tmq_conf_t{
char* clientId;
char* groupId;
char* ip;
uint16_t port;
tmq_commit_cb* commit_cb;
} tmq_conf_t;
struct tmq_t {
char groupId[256];
char clientId[256];
STscObj* pTscObj;
tmq_commit_cb* commit_cb;
SArray* clientTopics; // SArray<SMqClientTopic>
};
void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
conf->commit_cb = cb;
}
SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) {
tmq_t* pTmq = (void*)param;
SArray* pArray = taosArrayInit(0, sizeof(SKv));
if (pArray == NULL) {
return NULL;
}
SKv kv = {0};
kv.key = malloc(256);
if (kv.key == NULL) {
taosArrayDestroy(pArray);
return NULL;
}
strcpy(kv.key, "groupId");
kv.keyLen = strlen("groupId") + 1;
kv.value = malloc(256);
if (kv.value == NULL) {
free(kv.key);
taosArrayDestroy(pArray);
return NULL;
}
strcpy(kv.value, pTmq->groupId);
kv.valueLen = strlen(pTmq->groupId) + 1;
return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob);
taosArrayPush(pArray, &kv);
strcpy(kv.key, "clientUid");
kv.keyLen = strlen("clientUid") + 1;
*(uint32_t*)kv.value = pTmq->pTscObj->connId;
kv.valueLen = sizeof(uint32_t);
return NULL;
}
TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) {
STscObj* pTscObj = (STscObj*)taos;
SRequestObj* pRequest = NULL;
SQueryNode* pQuery = NULL;
SQueryDag* pDag = NULL;
char *dagStr = NULL;
tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
tmq_t* pTmq = malloc(sizeof(tmq_t));
if (pTmq == NULL) {
return NULL;
}
strcpy(pTmq->groupId, conf->groupId);
strcpy(pTmq->clientId, conf->clientId);
pTmq->pTscObj = (STscObj*)conn;
pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ;
return pTmq;
}
TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
STscObj *pTscObj = (STscObj*)taos;
SRequestObj *pRequest = NULL;
SQueryNode *pQueryNode = NULL;
char *pStr = NULL;
terrno = TSDB_CODE_SUCCESS;
if (taos == NULL || topicName == NULL || sql == NULL) {
tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql);
terrno = TSDB_CODE_TSC_INVALID_INPUT;
goto _return;
}
if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) {
tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1);
terrno = TSDB_CODE_TSC_INVALID_INPUT;
goto _return;
}
if (sqlLen > tsMaxSQLStringLen) {
tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
goto _return;
}
tscDebug("start to create topic, %s", topicName);
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
//temporary disabled until planner ready
#if 0
CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
//TODO: check sql valid
// todo check for invalid sql statement and return with error code
CHECK_CODE_GOTO(qCreateQueryDag(pQuery, &pDag), _return);
CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, pRequest->requestId), _return);
dagStr = qDagToString(pDag);
if(dagStr == NULL) {
//TODO
pStr = qDagToString(pRequest->body.pDag);
if(pStr == NULL) {
goto _return;
}
#endif
// The topic should be related to a database that the queried table is belonged to.
SName name = {0};
char dbName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&((SQueryStmtInfo*) pQueryNode)->pTableMetaInfo[0]->name, dbName);
tNameFromString(&name, dbName, T_NAME_ACCT|T_NAME_DB);
tNameFromString(&name, topicName, T_NAME_TABLE);
char topicFname[TSDB_TOPIC_FNAME_LEN] = {0};
tNameExtractFullName(&name, topicFname);
SCMCreateTopicReq req = {
.name = (char*)name,
.name = (char*) topicFname,
.igExists = 0,
/*.physicalPlan = dagStr,*/
.physicalPlan = (char*)sql,
.logicalPlan = "",
.physicalPlan = (char*) pStr,
.sql = (char*) sql,
.logicalPlan = "no logic plan",
};
int tlen = tSerializeSCMCreateTopicReq(NULL, &req);
......@@ -289,30 +418,54 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq
if(buf == NULL) {
goto _return;
}
void* abuf = buf;
tSerializeSCMCreateTopicReq(&abuf, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen };
pRequest->type = TDMT_MND_CREATE_TOPIC;
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, body);
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem);
_return:
qDestroyQuery(pQuery);
qDestroyQueryDag(pDag);
qDestroyQuery(pQueryNode);
if (body != NULL) {
destroySendMsgInfo(body);
}
if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
pRequest->code = terrno;
}
return pRequest;
}
typedef struct tmq_message_t {
int32_t numOfRows;
char* topicName;
TAOS_ROW row[];
} tmq_message_t;
tmq_message_t* tmq_consume_poll(tmq_t* mq, int64_t blocking_time) {
return NULL;
}
tmq_resp_err_t* tmq_commit(tmq_t* mq, void* callback, int32_t async) {
return NULL;
}
void tmq_message_destroy(tmq_message_t* mq_message) {
}
TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
STscObj *pTscObj = (STscObj *)taos;
if (sqlLen > (size_t) tsMaxSQLStringLen) {
......@@ -324,24 +477,22 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
nPrintTsc("%s", sql)
SRequestObj *pRequest = NULL;
SQueryNode *pQuery = NULL;
SQueryDag *pDag = NULL;
SQueryNode *pQueryNode = NULL;
terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
if (qIsDdlQuery(pQuery)) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
if (qIsDdlQuery(pQueryNode)) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return);
} else {
CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag), _return);
CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag), _return);
pRequest->code = terrno;
}
_return:
qDestroyQuery(pQuery);
qDestroyQueryDag(pDag);
qDestroyQuery(pQueryNode);
if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) {
pRequest->code = terrno;
}
......@@ -385,7 +536,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe
return 0;
}
STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) {
STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) {
STscObj *pTscObj = createTscObj(user, auth, db, pAppInfo);
if (NULL == pTscObj) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
......@@ -402,7 +553,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con
SMsgSendInfo* body = buildConnectMsg(pRequest);
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem);
if (pRequest->code != TSDB_CODE_SUCCESS) {
......@@ -413,7 +564,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con
taos_close(pTscObj);
pTscObj = NULL;
} else {
tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p, reqId:0x%"PRIx64, pTscObj->id, pTscObj->connId, pTscObj->pTransporter, pRequest->requestId);
tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p, reqId:0x%"PRIx64, pTscObj->id, pTscObj->connId, pTscObj->pAppInfo->pTransporter, pRequest->requestId);
destroyRequest(pRequest);
}
......@@ -444,7 +595,9 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) {
STscObj *pObj = pRequest->pTscObj;
char* db = getConnectionDB(pObj);
if (db != NULL) {
tstrncpy(pConnect->db, db, sizeof(pConnect->db));
}
tfree(db);
pConnect->pid = htonl(appInfo.pid);
......@@ -545,11 +698,15 @@ void* doFetchRow(SRequestObj* pRequest) {
if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) {
if (pRequest->type == TDMT_VND_QUERY) {
pRequest->type = TDMT_VND_FETCH;
// All data has returned to App already, no need to try again
if (pResultInfo->completed) {
return NULL;
}
scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData);
setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData);
pResultInfo->current = 0;
if (pResultInfo->numOfRows <= pResultInfo->current) {
if (pResultInfo->numOfRows == 0) {
return NULL;
}
......@@ -577,7 +734,7 @@ void* doFetchRow(SRequestObj* pRequest) {
int64_t transporterId = 0;
STscObj *pTscObj = pRequest->pTscObj;
asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem);
pRequest->type = TDMT_VND_SHOW_TABLES_FETCH;
......@@ -587,7 +744,7 @@ void* doFetchRow(SRequestObj* pRequest) {
int64_t transporterId = 0;
STscObj *pTscObj = pRequest->pTscObj;
asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem);
......@@ -611,12 +768,23 @@ _return:
return pResultInfo->row;
}
static void doPrepareResPtr(SReqResultInfo* pResInfo) {
if (pResInfo->row == NULL) {
pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES);
pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES);
pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t));
}
}
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) {
assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL);
if (numOfRows == 0) {
return;
}
// todo check for the failure of malloc
doPrepareResPtr(pResultInfo);
int32_t offset = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
pResultInfo->length[i] = pResultInfo->fields[i].bytes;
......@@ -629,9 +797,12 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t
char* getConnectionDB(STscObj* pObj) {
char *p = NULL;
pthread_mutex_lock(&pObj->mutex);
size_t len = strlen(pObj->db);
if (len > 0) {
p = strndup(pObj->db, tListLen(pObj->db));
pthread_mutex_unlock(&pObj->mutex);
}
pthread_mutex_unlock(&pObj->mutex);
return p;
}
......@@ -642,3 +813,14 @@ void setConnectionDB(STscObj* pTscObj, const char* db) {
pthread_mutex_unlock(&pTscObj->mutex);
}
void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
assert(pResultInfo != NULL && pRsp != NULL);
pResultInfo->pRspMsg = (const char*) pRsp;
pResultInfo->pData = (void*) pRsp->data;
pResultInfo->numOfRows = htonl(pRsp->numOfRows);
pResultInfo->current = 0;
pResultInfo->completed = (pRsp->completed == 1);
setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows);
}
source/client/src/clientMain.c
浏览文件 @ ae2d77e9
......@@ -265,7 +265,13 @@ const char *taos_data_type(int type) {
const char *taos_get_client_info() { return version; }
int taos_affected_rows(TAOS_RES *res) {
return ((SRequestObj*)res)->affectedRows;
if (res == NULL) {
return 0;
}
SRequestObj* pRequest = (SRequestObj*) res;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
return pResInfo->numOfRows;
}
int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; }
source/client/src/clientMsgHandler.c
浏览文件 @ ae2d77e9
......@@ -71,6 +71,9 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
pTscObj->pAppInfo->clusterId = pConnect->clusterId;
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY};
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);
// pRequest->body.resInfo.pRspMsg = pMsg->pData;
tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId,
pTscObj->pAppInfo->numOfConns);
......@@ -154,9 +157,6 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) {
pResInfo->fields = pFields;
pResInfo->numOfCols = pMetaMsg->numOfColumns;
pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES);
pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES);
pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t));
pRequest->body.showInfo.execId = pShow->showId;
......
source/client/test/clientTests.cpp
浏览文件 @ ae2d77e9
......@@ -53,6 +53,7 @@ TEST(testCase, connect_Test) {
if (pConn == NULL) {
printf("failed to connect to server, reason:%s\n", taos_errstr(NULL));
}
sleep(3);
taos_close(pConn);
}
......@@ -148,27 +149,27 @@ TEST(testCase, connect_Test) {
//}
//
//TEST(testCase, create_db_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// ASSERT_TRUE(pFields == NULL);
//
// int32_t numOfFields = taos_num_fields(pRes);
// ASSERT_EQ(numOfFields, 0);
//
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create database abc1 vgroups 4");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
// taos_close(pConn);
//TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
//assert(pConn != NULL);
//TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
//if (taos_errno(pRes) != 0) {
//printf("error in create db, reason:%s\n", taos_errstr(pRes));
//}
//TAOS_FIELD* pFields = taos_fetch_fields(pRes);
//ASSERT_TRUE(pFields == NULL);
//int32_t numOfFields = taos_num_fields(pRes);
//ASSERT_EQ(numOfFields, 0);
//taos_free_result(pRes);
//pRes = taos_query(pConn, "create database abc1 vgroups 4");
//if (taos_errno(pRes) != 0) {
//printf("error in create db, reason:%s\n", taos_errstr(pRes));
//}
//taos_close(pConn);
//}
//
//TEST(testCase, create_dnode_Test) {
......@@ -194,7 +195,7 @@ TEST(testCase, connect_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2");
// TAOS_RES* pRes = taos_query(pConn, "drop dnode 3");
// if (taos_errno(pRes) != 0) {
// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes));
// }
......@@ -205,6 +206,11 @@ TEST(testCase, connect_Test) {
// int32_t numOfFields = taos_num_fields(pRes);
// ASSERT_EQ(numOfFields, 0);
//
// pRes = taos_query(pConn, "drop dnode 4");
// if (taos_errno(pRes) != 0) {
// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes));
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
......@@ -227,45 +233,39 @@ TEST(testCase, connect_Test) {
// taos_close(pConn);
//}
//
//// TEST(testCase, drop_db_test) {
//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
//// assert(pConn != NULL);
////
//// showDB(pConn);
////
//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1");
//// if (taos_errno(pRes) != 0) {
//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes));
//// }
//// taos_free_result(pRes);
////
//// showDB(pConn);
////
//// pRes = taos_query(pConn, "create database abc1");
//// if (taos_errno(pRes) != 0) {
//// printf("create to drop db, reason:%s\n", taos_errstr(pRes));
//// }
//// taos_free_result(pRes);
//// taos_close(pConn);
////}
//
//TEST(testCase, create_stable_Test) {
// TEST(testCase, drop_db_test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
// showDB(pConn);
//
// TAOS_RES* pRes = taos_query(pConn, "drop database abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// printf("failed to drop db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "use abc1");
// showDB(pConn);
//
// pRes = taos_query(pConn, "create database abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in use db, reason:%s\n", taos_errstr(pRes));
// printf("create to drop db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
// taos_close(pConn);
//}
//TEST(testCase, create_stable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
// pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)");
// if (taos_errno(pRes) != 0) {
// printf("error in create stable, reason:%s\n", taos_errstr(pRes));
// }
......@@ -277,22 +277,40 @@ TEST(testCase, connect_Test) {
// ASSERT_EQ(numOfFields, 0);
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, create_table_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// taos_free_result(pRes);
// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
// if (taos_errno(pRes) != 0) {
// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
// }
//
// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)");
// taos_free_result(pRes);
// pRes = taos_query(pConn, "drop stable `123_$^)`");
// if (taos_errno(pRes) != 0) {
// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
// }
//
// taos_close(pConn);
//}
//
TEST(testCase, create_table_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)");
ASSERT_EQ(taos_errno(pRes), 0);
taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)");
ASSERT_NE(taos_errno(pRes), 0);
taos_free_result(pRes);
taos_close(pConn);
}
//TEST(testCase, create_ctable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
......@@ -311,37 +329,37 @@ TEST(testCase, connect_Test) {
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, show_stable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
TEST(testCase, show_stable_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != nullptr);
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("failed to use db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "show stables");
// if (taos_errno(pRes) != 0) {
// printf("failed to show stables, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
TAOS_RES* pRes = taos_query(pConn, "show abc1.stables");
if (taos_errno(pRes) != 0) {
printf("failed to show stables, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
}
//
//TEST(testCase, show_vgroup_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
......@@ -499,43 +517,34 @@ TEST(testCase, connect_Test) {
//
// taosHashCleanup(phash);
//}
// TEST(testCase, create_topic_Test) {
//
//TEST(testCase, create_topic_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "create database abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "use abc1");
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in use db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
// if (taos_errno(pRes) != 0) {
// printf("error in create stable, reason:%s\n", taos_errstr(pRes));
// }
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// ASSERT_TRUE(pFields == NULL);
// ASSERT_TRUE(pFields == nullptr);
//
// int32_t numOfFields = taos_num_fields(pRes);
// ASSERT_EQ(numOfFields, 0);
//
// taos_free_result(pRes);
//
// char* sql = "select * from st1";
// tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql));
// char* sql = "select * from tu";
// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql));
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, insert_test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// ASSERT_EQ(pConn, nullptr);
// ASSERT_NE(pConn, nullptr);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// taos_free_result(pRes);
......@@ -550,49 +559,82 @@ TEST(testCase, connect_Test) {
// taos_free_result(pRes);
// taos_close(pConn);
//}
//#endif
TEST(testCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
//TEST(testCase, projection_query_tables) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// ASSERT_NE(pConn, nullptr);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// taos_free_result(pRes);
//
//// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
//// if (taos_errno(pRes) != 0) {
//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
//// }
//// taos_free_result(pRes);
////
//// pRes = taos_query(pConn, "create table tu using st1 tags(1)");
//// if (taos_errno(pRes) != 0) {
//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
//// }
//// taos_free_result(pRes);
////
//// for(int32_t i = 0; i < 100; ++i) {
//// char sql[512] = {0};
//// sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
//// TAOS_RES* p = taos_query(pConn, sql);
//// if (taos_errno(p) != 0) {
//// printf("failed to insert data, reason:%s\n", taos_errstr(p));
//// }
////
//// taos_free_result(p);
//// }
//
// pRes = taos_query(pConn, "select * from tu");
// if (taos_errno(pRes) != 0) {
// printf("failed to use db, reason:%s\n", taos_errstr(pRes));
// printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// return;
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
TAOS_RES* pRes = taos_query(pConn, "use abc1");
// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)");
taos_free_result(pRes);
//TEST(testCase, projection_query_stables) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// ASSERT_NE(pConn, nullptr);
//
// pRes = taos_query(pConn, "create table tu using m1 tags(1)");
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "insert into tu values(now, 1)");
// pRes = taos_query(pConn, "select ts,k from m1");
// if (taos_errno(pRes) != 0) {
// printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
pRes = taos_query(pConn, "select * from tu");
if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
}
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
source/common/src/tmsg.c
浏览文件 @ ae2d77e9
......@@ -89,7 +89,7 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) {
int tlen = 0;
tlen += taosEncodeSClientHbKey(buf, &pReq->connKey);
int kvNum = taosHashGetSize(pReq->info);
int32_t kvNum = taosHashGetSize(pReq->info);
tlen += taosEncodeFixedI32(buf, kvNum);
SKv kv;
void* pIter = taosHashIterate(pReq->info, pIter);
......@@ -104,14 +104,15 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) {
return tlen;
}
void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) {
ASSERT(pReq->info != NULL);
void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) {
buf = taosDecodeSClientHbKey(buf, &pReq->connKey);
// TODO: error handling
int kvNum;
taosDecodeFixedI32(buf, &kvNum);
int32_t kvNum;
buf = taosDecodeFixedI32(buf, &kvNum);
if (pReq->info == NULL) {
pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
}
for(int i = 0; i < kvNum; i++) {
SKv kv;
buf = taosDecodeSKv(buf, &kv);
......@@ -121,12 +122,69 @@ void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) {
return buf;
}
int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq) {
int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp) {
int tlen = 0;
tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey);
tlen += taosEncodeFixedI32(buf, pRsp->status);
tlen += taosEncodeFixedI32(buf, pRsp->bodyLen);
tlen += taosEncodeBinary(buf, pRsp->body, pRsp->bodyLen);
return tlen;
}
void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp) {
buf = taosDecodeSClientHbKey(buf, &pRsp->connKey);
buf = taosDecodeFixedI32(buf, &pRsp->status);
buf = taosDecodeFixedI32(buf, &pRsp->bodyLen);
buf = taosDecodeBinary(buf, &pRsp->body, pRsp->bodyLen);
return buf;
}
int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pBatchReq) {
int tlen = 0;
tlen += taosEncodeFixedI64(buf, pBatchReq->reqId);
int32_t reqNum = taosArrayGetSize(pBatchReq->reqs);
tlen += taosEncodeFixedI32(buf, reqNum);
for (int i = 0; i < reqNum; i++) {
SClientHbReq* pReq = taosArrayGet(pBatchReq->reqs, i);
tlen += tSerializeSClientHbReq(buf, pReq);
}
return tlen;
}
void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq) {
void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) {
buf = taosDecodeFixedI64(buf, &pBatchReq->reqId);
if (pBatchReq->reqs == NULL) {
pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq));
}
int32_t reqNum;
buf = taosDecodeFixedI32(buf, &reqNum);
for (int i = 0; i < reqNum; i++) {
SClientHbReq req = {0};
buf = tDeserializeSClientHbReq(buf, &req);
taosArrayPush(pBatchReq->reqs, &req);
}
return buf;
}
int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp) {
int tlen = 0;
int32_t sz = taosArrayGetSize(pBatchRsp->rsps);
tlen += taosEncodeFixedI32(buf, sz);
for (int i = 0; i < sz; i++) {
SClientHbRsp* pRsp = taosArrayGet(pBatchRsp->rsps, i);
tlen += tSerializeSClientHbRsp(buf, pRsp);
}
return tlen;
}
void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp) {
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp));
for (int i = 0; i < sz; i++) {
SClientHbRsp rsp = {0};
buf = tDeserializeSClientHbRsp(buf, &rsp);
taosArrayPush(pBatchRsp->rsps, &rsp);
}
return buf;
}
......
source/dnode/mgmt/impl/src/dndEnv.c
浏览文件 @ ae2d77e9
......@@ -293,7 +293,7 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) {
if (vnodeInit(&vnodeOpt) != 0) {
dError("failed to init vnode since %s", terrstr());
dndCleanup();
return NULL;
return -1;
}
memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg));
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ ae2d77e9
......@@ -350,7 +350,7 @@ typedef struct SMqTopicObj {
// TODO: add cache and change name to id
typedef struct SMqConsumerTopic {
char name[TSDB_TOPIC_FNAME_LEN];
char name[TSDB_TOPIC_NAME_LEN];
SList *vgroups; // SList<int32_t>
} SMqConsumerTopic;
......@@ -409,7 +409,7 @@ typedef struct SMqVGroupHbObj {
#if 0
typedef struct SCGroupObj {
char name[TSDB_TOPIC_FNAME_LEN];
char name[TSDB_TOPIC_NAME_LEN];
int64_t createTime;
int64_t updateTime;
uint64_t uid;
......
source/dnode/mnode/impl/src/mndProfile.c
浏览文件 @ ae2d77e9
......@@ -258,6 +258,39 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) {
}
static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
SMnode *pMnode = pReq->pMnode;
char *batchReqStr = pReq->rpcMsg.pCont;
SClientHbBatchReq batchReq = {0};
tDeserializeSClientHbBatchReq(batchReqStr, &batchReq);
SArray *pArray = batchReq.reqs;
int sz = taosArrayGetSize(pArray);
SClientHbBatchRsp batchRsp = {0};
batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp));
for (int i = 0; i < sz; i++) {
SClientHbReq* pHbReq = taosArrayGet(pArray, i);
if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) {
} else if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_MQ) {
SClientHbRsp rsp = {
.status = 0,
.connKey = pHbReq->connKey,
.bodyLen = 0,
.body = NULL
};
taosArrayPush(batchRsp.rsps, &rsp);
}
}
int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp);
void* buf = rpcMallocCont(tlen);
void* bufCopy = buf;
tSerializeSClientHbBatchRsp(&bufCopy, &batchRsp);
pReq->contLen = tlen;
pReq->pCont = buf;
return 0;
#if 0
SMnode *pMnode = pReq->pMnode;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
......@@ -327,6 +360,7 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
pReq->contLen = sizeof(SConnectRsp);
pReq->pCont = pRsp;
return 0;
#endif
}
static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) {
......
source/dnode/mnode/impl/src/mndTopic.c
浏览文件 @ ae2d77e9
......@@ -118,11 +118,13 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER);
pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char));
SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER);
// SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
// SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER);
// SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
// SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER)
......@@ -178,7 +180,7 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) {
SName name = {0};
tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TOPIC);
tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
char db[TSDB_TABLE_FNAME_LEN] = {0};
tNameGetFullDbName(&name, db);
......@@ -203,20 +205,24 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
return pDrop;
}
static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *pCreate) {
static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *creattopReq) {
// deserialize and other stuff
return 0;
}
static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
SMqTopicObj topicObj = {0};
tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
topicObj.createTime = taosGetTimestampMs();
topicObj.updateTime = topicObj.createTime;
topicObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
topicObj.dbUid = pDb->uid;
topicObj.version = 1;
topicObj.sql = strdup(pCreate->sql);
topicObj.physicalPlan = strdup(pCreate->physicalPlan);
topicObj.logicalPlan = strdup(pCreate->logicalPlan);
topicObj.sqlLen = strlen(pCreate->sql);
SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj);
if (pTopicRaw == NULL) return -1;
......@@ -228,46 +234,47 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq
static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
char *msgStr = pMsg->rpcMsg.pCont;
SCMCreateTopicReq *pCreate;
tDeserializeSCMCreateTopicReq(msgStr, pCreate);
mDebug("topic:%s, start to create", pCreate->name);
SCMCreateTopicReq createTopicReq = {0};
tDeserializeSCMCreateTopicReq(msgStr, &createTopicReq);
mDebug("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql);
if (mndCheckCreateTopicMsg(pCreate) != 0) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
if (mndCheckCreateTopicMsg(&createTopicReq) != 0) {
mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr());
return -1;
}
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pCreate->name);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, createTopicReq.name);
if (pTopic != NULL) {
sdbRelease(pMnode->pSdb, pTopic);
if (pCreate->igExists) {
mDebug("topic:%s, already exist, ignore exist is set", pCreate->name);
if (createTopicReq.igExists) {
mDebug("topic:%s, already exist, ignore exist is set", createTopicReq.name);
return 0;
} else {
terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST;
mError("db:%s, failed to create since %s", pCreate->name, terrstr());
mError("db:%s, failed to create since %s", createTopicReq.name, terrstr());
return -1;
}
}
SDbObj *pDb = mndAcquireDbByTopic(pMnode, pCreate->name);
SDbObj *pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr());
return -1;
}
int32_t code = mndCreateTopic(pMnode, pMsg, pCreate, pDb);
int32_t code = mndCreateTopic(pMnode, pMsg, &createTopicReq, pDb);
mndReleaseDb(pMnode, pDb);
if (code != 0) {
terrno = code;
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr());
return -1;
}
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
return TSDB_CODE_SUCCESS;
}
static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic) { return 0; }
......
source/dnode/mnode/impl/test/profile/profile.cpp
浏览文件 @ ae2d77e9
......@@ -96,6 +96,38 @@ TEST_F(MndTestProfile, 03_ConnectMsg_Show) {
}
TEST_F(MndTestProfile, 04_HeartBeatMsg) {
SClientHbBatchReq batchReq;
batchReq.reqs = taosArrayInit(0, sizeof(SClientHbReq));
SClientHbReq req = {0};
req.connKey = {.connId = 123, .hbType = HEARTBEAT_TYPE_MQ};
req.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
SKv kv;
kv.key = (void*)"abc";
kv.keyLen = 4;
kv.value = (void*)"bcd";
kv.valueLen = 4;
taosHashPut(req.info, kv.key, kv.keyLen, kv.value, kv.valueLen);
taosArrayPush(batchReq.reqs, &req);
int32_t tlen = tSerializeSClientHbBatchReq(NULL, &batchReq);
void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen);
void* bufCopy = buf;
tSerializeSClientHbBatchReq(&bufCopy, &batchReq);
SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, buf, tlen);
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
char* pRspChar = (char*)pMsg->pCont;
SClientHbBatchRsp rsp = {0};
tDeserializeSClientHbBatchRsp(pRspChar, &rsp);
int sz = taosArrayGetSize(rsp.rsps);
ASSERT_EQ(sz, 1);
SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0);
EXPECT_EQ(pRsp->connKey.connId, 123);
EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ);
EXPECT_EQ(pRsp->status, 0);
#if 0
int32_t contLen = sizeof(SHeartBeatReq);
SHeartBeatReq* pReq = (SHeartBeatReq*)rpcMallocCont(contLen);
......@@ -129,9 +161,12 @@ TEST_F(MndTestProfile, 04_HeartBeatMsg) {
EXPECT_EQ(pRsp->epSet.numOfEps, 1);
EXPECT_EQ(pRsp->epSet.port[0], 9031);
EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost");
#endif
}
TEST_F(MndTestProfile, 05_KillConnMsg) {
// temporary remove since kill will use new heartbeat msg
#if 0
{
int32_t contLen = sizeof(SKillConnReq);
......@@ -190,6 +225,7 @@ TEST_F(MndTestProfile, 05_KillConnMsg) {
connId = pRsp->connId;
}
#endif
}
TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) {
......@@ -204,6 +240,8 @@ TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) {
}
TEST_F(MndTestProfile, 07_KillQueryMsg) {
// temporary remove since kill will use new heartbeat msg
#if 0
{
int32_t contLen = sizeof(SKillQueryReq);
......@@ -252,6 +290,7 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) {
EXPECT_EQ(pRsp->epSet.port[0], 9031);
EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost");
}
#endif
}
TEST_F(MndTestProfile, 08_KillQueryMsg_InvalidConn) {
......
source/dnode/vnode/CMakeLists.txt
浏览文件 @ ae2d77e9
add_subdirectory(meta)
add_subdirectory(tq)
add_subdirectory(tsdb)
add_subdirectory(impl)
\ No newline at end of file
aux_source_directory(src/meta META_SRC)
aux_source_directory(src/tq TQ_SRC)
aux_source_directory(src/tsdb TSDB_SRC)
aux_source_directory(src/vnd VND_SRC)
list(APPEND
VNODE_SRC
${META_SRC}
${TQ_SRC}
${TSDB_SRC}
${VND_SRC}
)
add_library(vnode STATIC ${VNODE_SRC})
target_include_directories(
vnode
PUBLIC inc
PRIVATE src/inc
)
target_link_libraries(
vnode
PUBLIC os
PUBLIC util
PUBLIC common
PUBLIC transport
PUBLIC bdb
PUBLIC tfs
PUBLIC wal
PUBLIC qworker
)
if(${BUILD_TEST})
# add_subdirectory(test)
endif(${BUILD_TEST})
source/dnode/vnode/impl/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ de8b6e73
aux_source_directory(src VNODE_SRC)
add_library(vnode STATIC ${VNODE_SRC})
target_include_directories(
vnode
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode"
private "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
vnode
PUBLIC os
PUBLIC transport
PUBLIC meta
PUBLIC tq
PUBLIC tsdb
PUBLIC wal
PUBLIC sync
PUBLIC cjson
PUBLIC qworker
)
# test
if(${BUILD_TEST})
# add_subdirectory(test)
endif(${BUILD_TEST})
source/dnode/vnode/impl/inc/vnodeRead.h 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_READ_H_
#define _TD_VNODE_READ_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "vnodeInt.h"
void vnodeProcessReadMsg(SVnode *pVnode, SVnodeMsg *pMsg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_READ_H_*/
source/dnode/vnode/impl/inc/vnodeRequest.h 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_REQUEST_H_
#define _TD_VNODE_REQUEST_H_
#include "vnode.h"
#ifdef __cplusplus
extern "C" {
#endif
// SVDropTbReq
// int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq);
// void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_REQUEST_H_*/
\ No newline at end of file
source/dnode/vnode/impl/src/vnodeFS.c 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeDef.h"
\ No newline at end of file
source/dnode/vnode/impl/src/vnodeRead.c 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeDef.h"
\ No newline at end of file
source/dnode/vnode/impl/src/vnodeRequest.c 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeDef.h"
#if 0
static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq);
static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq);
int vnodeBuildReq(void **buf, const SVnodeReq *pReq, tmsg_t type) {
int tsize = 0;
tsize += taosEncodeFixedU64(buf, pReq->ver);
switch (type) {
case TDMT_VND_CREATE_STB:
tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq));
break;
case TDMT_VND_SUBMIT:
/* code */
break;
default:
break;
}
/* TODO */
return tsize;
}
void *vnodeParseReq(void *buf, SVnodeReq *pReq, tmsg_t type) {
buf = taosDecodeFixedU64(buf, &(pReq->ver));
switch (type) {
case TDMT_VND_CREATE_STB:
buf = vnodeParseCreateTableReq(buf, &(pReq->ctReq));
break;
default:
break;
}
// TODO
return buf;
}
static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq) {
int tsize = 0;
tsize += taosEncodeString(buf, pReq->name);
tsize += taosEncodeFixedU32(buf, pReq->ttl);
tsize += taosEncodeFixedU32(buf, pReq->keep);
tsize += taosEncodeFixedU8(buf, pReq->type);
switch (pReq->type) {
case META_SUPER_TABLE:
tsize += taosEncodeFixedU64(buf, pReq->stbCfg.suid);
tsize += tdEncodeSchema(buf, pReq->stbCfg.pSchema);
tsize += tdEncodeSchema(buf, pReq->stbCfg.pTagSchema);
break;
case META_CHILD_TABLE:
tsize += taosEncodeFixedU64(buf, pReq->ctbCfg.suid);
tsize += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
break;
case META_NORMAL_TABLE:
tsize += tdEncodeSchema(buf, pReq->ntbCfg.pSchema);
break;
default:
break;
}
return tsize;
}
static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq) {
buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeFixedU32(buf, &(pReq->ttl));
buf = taosDecodeFixedU32(buf, &(pReq->keep));
buf = taosDecodeFixedU8(buf, &(pReq->type));
switch (pReq->type) {
case META_SUPER_TABLE:
buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid));
buf = tdDecodeSchema(buf, &(pReq->stbCfg.pSchema));
buf = tdDecodeSchema(buf, &(pReq->stbCfg.pTagSchema));
break;
case META_CHILD_TABLE:
buf = taosDecodeFixedU64(buf, &(pReq->ctbCfg.suid));
buf = tdDecodeKVRow(buf, &(pReq->ctbCfg.pTag));
break;
case META_NORMAL_TABLE:
buf = tdDecodeSchema(buf, &(pReq->ntbCfg.pSchema));
break;
default:
break;
}
return buf;
}
int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq) {
// TODO
return 0;
}
void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq) {
// TODO
}
#endif
\ No newline at end of file
source/dnode/vnode/impl/src/vnodeSync.c 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
\ No newline at end of file
source/dnode/vnode/impl/test/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ de8b6e73
# Vnode API test
add_executable(vnodeApiTests "")
target_sources(vnodeApiTests
PRIVATE
"vnodeApiTests.cpp"
)
target_link_libraries(vnodeApiTests vnode gtest gtest_main)
add_test(
NAME vnode_api_tests
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests
)
\ No newline at end of file
source/dnode/vnode/impl/test/vBenchmarkTest.cpp 已删除 100644 → 0
浏览文件 @ de8b6e73
// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest
// https://github.com/google/benchmark
\ No newline at end of file
source/dnode/vnode/impl/test/vnodeApiTests.cpp 已删除 100644 → 0
浏览文件 @ de8b6e73
/**
* @file vnodeApiTests.cpp
* @author hzcheng (hzcheng@taosdata.com)
* @brief VNODE module API tests
* @version 0.1
* @date 2021-12-13
*
* @copyright Copyright (c) 2021
*
*/
#include <gtest/gtest.h>
#include <iostream>
#include "vnode.h"
static STSchema *vtCreateBasicSchema() {
STSchemaBuilder sb;
STSchema * pSchema = NULL;
tdInitTSchemaBuilder(&sb, 0);
tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0);
for (int i = 1; i < 10; i++) {
tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, i, 0);
}
pSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
return pSchema;
}
static STSchema *vtCreateBasicTagSchema() {
STSchemaBuilder sb;
STSchema * pSchema = NULL;
tdInitTSchemaBuilder(&sb, 0);
tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0);
for (int i = 10; i < 12; i++) {
tdAddColToSchema(&sb, TSDB_DATA_TYPE_BINARY, i, 20);
}
pSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
return pSchema;
}
static SKVRow vtCreateBasicTag() {
SKVRowBuilder rb;
SKVRow pTag;
tdInitKVRowBuilder(&rb);
for (int i = 0; i < 2; i++) {
void *pVal = malloc(sizeof(VarDataLenT) + strlen("foo"));
varDataLen(pVal) = strlen("foo");
memcpy(varDataVal(pVal), "foo", strlen("foo"));
tdAddColToKVRow(&rb, i, TSDB_DATA_TYPE_BINARY, pVal);
free(pVal);
}
pTag = tdGetKVRowFromBuilder(&rb);
tdDestroyKVRowBuilder(&rb);
return pTag;
}
static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) {
SRpcMsg * pMsg;
STSchema *pSchema;
STSchema *pTagSchema;
int zs;
void * pBuf;
pSchema = vtCreateBasicSchema();
pTagSchema = vtCreateBasicTagSchema();
SVnodeReq vCreateSTbReq;
vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema);
zs = vnodeBuildReq(NULL, &vCreateSTbReq, TDMT_VND_CREATE_STB);
pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs);
pMsg->msgType = TDMT_VND_CREATE_STB;
pMsg->contLen = zs;
pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg));
pBuf = pMsg->pCont;
vnodeBuildReq(&pBuf, &vCreateSTbReq, TDMT_VND_CREATE_STB);
META_CLEAR_TB_CFG(&vCreateSTbReq);
tdFreeSchema(pSchema);
tdFreeSchema(pTagSchema);
*ppMsg = pMsg;
}
static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) {
SRpcMsg *pMsg;
int tz;
SKVRow pTag = vtCreateBasicTag();
SVnodeReq vCreateCTbReq;
vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag);
tz = vnodeBuildReq(NULL, &vCreateCTbReq, TDMT_VND_CREATE_TABLE);
pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz);
pMsg->msgType = TDMT_VND_CREATE_TABLE;
pMsg->contLen = tz;
pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg));
void *pBuf = pMsg->pCont;
vnodeBuildReq(&pBuf, &vCreateCTbReq, TDMT_VND_CREATE_TABLE);
META_CLEAR_TB_CFG(&vCreateCTbReq);
free(pTag);
*ppMsg = pMsg;
}
static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) {
// TODO
}
static void vtBuildSubmitReq(SRpcMsg **ppMsg) {
SRpcMsg * pMsg;
SSubmitMsg *pSubmitMsg;
SSubmitBlk *pSubmitBlk;
int tz = 1024; // TODO
pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz);
pMsg->msgType = TDMT_VND_SUBMIT;
pMsg->contLen = tz;
pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg));
// For submit msg header
pSubmitMsg = (SSubmitMsg *)(pMsg->pCont);
// pSubmitMsg->header.contLen = 0;
// pSubmitMsg->header.vgId = 0;
// pSubmitMsg->length = 0;
pSubmitMsg->numOfBlocks = 1;
// For submit blk
pSubmitBlk = (SSubmitBlk *)(pSubmitMsg->blocks);
pSubmitBlk->uid = 0;
pSubmitBlk->tid = 0;
pSubmitBlk->padding = 0;
pSubmitBlk->sversion = 0;
pSubmitBlk->dataLen = 0;
pSubmitBlk->numOfRows = 0;
// For row batch
*ppMsg = pMsg;
}
static void vtClearMsgBatch(SArray *pMsgArr) {
SRpcMsg *pMsg;
for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) {
pMsg = *(SRpcMsg **)taosArrayGet(pMsgArr, i);
free(pMsg);
}
taosArrayClear(pMsgArr);
}
static void vtProcessAndApplyReqs(SVnode *pVnode, SArray *pMsgArr) {
int rcode;
SRpcMsg *pReq;
SRpcMsg *pRsp;
rcode = vnodeProcessWMsgs(pVnode, pMsgArr);
GTEST_ASSERT_EQ(rcode, 0);
for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) {
pReq = *(SRpcMsg **)taosArrayGet(pMsgArr, i);
rcode = vnodeApplyWMsg(pVnode, pReq, NULL);
GTEST_ASSERT_EQ(rcode, 0);
}
}
TEST(vnodeApiTest, vnode_simple_create_table_test) {
tb_uid_t suid = 1638166374163;
SRpcMsg *pMsg;
SArray * pMsgArr = NULL;
SVnode * pVnode;
int rcode;
int ntables = 1000000;
int batch = 10;
char tbname[128];
pMsgArr = (SArray *)taosArrayInit(batch, sizeof(pMsg));
vnodeDestroy("vnode1");
GTEST_ASSERT_GE(vnodeInit(2), 0);
// CREATE AND OPEN A VNODE
pVnode = vnodeOpen("vnode1", NULL);
ASSERT_NE(pVnode, nullptr);
// CREATE A SUPER TABLE
sprintf(tbname, "st");
vtBuildCreateStbReq(suid, tbname, &pMsg);
taosArrayPush(pMsgArr, &pMsg);
vtProcessAndApplyReqs(pVnode, pMsgArr);
vtClearMsgBatch(pMsgArr);
// CREATE A LOT OF CHILD TABLES
for (int i = 0; i < ntables / batch; i++) {
// Build request batch
for (int j = 0; j < batch; j++) {
sprintf(tbname, "ct%d", i * batch + j + 1);
vtBuildCreateCtbReq(suid, tbname, &pMsg);
taosArrayPush(pMsgArr, &pMsg);
}
// Process request batch
vtProcessAndApplyReqs(pVnode, pMsgArr);
// Clear request batch
vtClearMsgBatch(pMsgArr);
}
// CLOSE THE VNODE
vnodeClose(pVnode);
vnodeCleanup();
taosArrayDestroy(pMsgArr);
}
TEST(vnodeApiTest, vnode_simple_insert_test) {
const char *vname = "vnode2";
char tbname[128];
tb_uid_t suid = 1638166374163;
SRpcMsg * pMsg;
SArray * pMsgArr;
int rcode;
SVnode * pVnode;
int batch = 1;
int loop = 1000000;
pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg));
vnodeDestroy(vname);
GTEST_ASSERT_GE(vnodeInit(2), 0);
// Open a vnode
pVnode = vnodeOpen(vname, NULL);
GTEST_ASSERT_NE(pVnode, nullptr);
// 1. CREATE A SUPER TABLE
sprintf(tbname, "st");
vtBuildCreateStbReq(suid, tbname, &pMsg);
taosArrayPush(pMsgArr, &pMsg);
vtProcessAndApplyReqs(pVnode, pMsgArr);
vtClearMsgBatch(pMsgArr);
// 2. CREATE A CHILD TABLE
sprintf(tbname, "t0");
vtBuildCreateCtbReq(suid, tbname, &pMsg);
taosArrayPush(pMsgArr, &pMsg);
vtProcessAndApplyReqs(pVnode, pMsgArr);
vtClearMsgBatch(pMsgArr);
// 3. WRITE A LOT OF TIME-SERIES DATA
for (int j = 0; j < loop; j++) {
for (int i = 0; i < batch; i++) {
vtBuildSubmitReq(&pMsg);
taosArrayPush(pMsgArr, &pMsg);
}
vtProcessAndApplyReqs(pVnode, pMsgArr);
vtClearMsgBatch(pMsgArr);
}
// Close the vnode
vnodeClose(pVnode);
vnodeCleanup();
taosArrayDestroy(pMsgArr);
}
\ No newline at end of file
include/dnode/vnode/vnode.h → source/dnode/vnode/inc/vnode.h
浏览文件 @ ae2d77e9
......@@ -36,45 +36,25 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq);
typedef struct SVnodeCfg {
int32_t vgId;
SDnode *pDnode;
/** vnode buffer pool options */
struct {
/** write buffer size */
SDnode * pDnode;
uint64_t wsize;
uint64_t ssize;
uint64_t lsize;
/** use heap allocator or arena allocator */
bool isHeapAllocator;
};
/** time to live of tables in this vnode */
uint32_t ttl;
/** data to keep in this vnode */
uint32_t keep;
/** if TS data is eventually consistency */
bool isWeak;
/** TSDB config */
STsdbCfg tsdbCfg;
/** META config */
SMetaCfg metaCfg;
/** TQ config */
STqCfg tqCfg;
/** WAL config */
SWalCfg walCfg;
} SVnodeCfg;
typedef struct {
int32_t sver;
char *timezone;
char *locale;
char *charset;
char * timezone;
char * locale;
char * charset;
uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO)
PutReqToVQueryQFp putReqToVQueryQFp;
} SVnodeOpt;
......
source/dnode/vnode/meta/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ de8b6e73
set(META_DB_IMPL_LIST "BDB" "SQLITE")
set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation")
set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST})
if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)
message(STATUS "META DB Impl: ${META_DB_IMPL}==============")
else()
message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============")
endif()
aux_source_directory(src META_SRC)
if(${META_DB_IMPL} STREQUAL "BDB")
list(REMOVE_ITEM META_SRC "src/metaSQLiteImpl.c")
elseif(${META_DB_IMPL} STREQUAL "SQLITE")
list(REMOVE_ITEM META_SRC "src/metaBDBImpl.c")
endif()
add_library(meta STATIC ${META_SRC})
target_include_directories(
meta
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/meta"
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
meta
PUBLIC common
PUBLIC index
)
if(${META_DB_IMPL} STREQUAL "BDB")
target_link_libraries(
meta
PUBLIC bdb
)
elseif(${META_DB_IMPL} STREQUAL "SQLITE")
target_link_libraries(
meta
PUBLIC sqlite
)
endif()
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})
source/dnode/vnode/meta/src/metaSQLiteImpl.c 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "metaDef.h"
#include "sqlite3.h"
struct SMetaDB {
sqlite3 *pDB;
};
int metaOpenDB(SMeta *pMeta) {
char dir[128];
int rc;
char *err = NULL;
pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB));
if (pMeta->pDB == NULL) {
// TODO: handle error
return -1;
}
sprintf(dir, "%s/meta.db", pMeta->path);
rc = sqlite3_open(dir, &(pMeta->pDB->pDB));
if (rc != SQLITE_OK) {
// TODO: handle error
printf("failed to open meta.db\n");
}
// For all tables
rc = sqlite3_exec(pMeta->pDB->pDB,
"CREATE TABLE IF NOT EXISTS tb ("
" tbname VARCHAR(256) NOT NULL UNIQUE,"
" tb_uid INTEGER NOT NULL UNIQUE "
");",
NULL, NULL, &err);
if (rc != SQLITE_OK) {
// TODO: handle error
printf("failed to create meta table tb since %s\n", err);
}
// For super tables
rc = sqlite3_exec(pMeta->pDB->pDB,
"CREATE TABLE IF NOT EXISTS stb ("
" tb_uid INTEGER NOT NULL UNIQUE,"
" tbname VARCHAR(256) NOT NULL UNIQUE,"
" tb_schema BLOB NOT NULL,"
" tag_schema BLOB NOT NULL"
");",
NULL, NULL, &err);
if (rc != SQLITE_OK) {
// TODO: handle error
printf("failed to create meta table stb since %s\n", err);
}
// For normal tables
rc = sqlite3_exec(pMeta->pDB->pDB,
"CREATE TABLE IF NOT EXISTS ntb ("
" tb_uid INTEGER NOT NULL UNIQUE,"
" tbname VARCHAR(256) NOT NULL,"
" tb_schema BLOB NOT NULL"
");",
NULL, NULL, &err);
if (rc != SQLITE_OK) {
// TODO: handle error
printf("failed to create meta table ntb since %s\n", err);
}
sqlite3_exec(pMeta->pDB->pDB, "BEGIN;", NULL, NULL, &err);
tfree(err);
return 0;
}
void metaCloseDB(SMeta *pMeta) {
if (pMeta->pDB) {
sqlite3_exec(pMeta->pDB->pDB, "COMMIT;", NULL, NULL, NULL);
sqlite3_close(pMeta->pDB->pDB);
free(pMeta->pDB);
pMeta->pDB = NULL;
}
// TODO
}
int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) {
char sql[256];
char * err = NULL;
int rc;
tb_uid_t uid;
sqlite3_stmt *stmt;
char buf[256];
void * pBuf;
switch (pTbCfg->type) {
case META_SUPER_TABLE:
uid = pTbCfg->stbCfg.suid;
sprintf(sql,
"INSERT INTO tb VALUES (\'%s\', %" PRIu64
");"
"CREATE TABLE IF NOT EXISTS stb_%" PRIu64
" ("
" tb_uid INTEGER NOT NULL UNIQUE,"
" tbname VARCHAR(256),"
" tag1 INTEGER);",
pTbCfg->name, uid, uid);
rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err);
if (rc != SQLITE_OK) {
printf("failed to create normal table since %s\n", err);
}
sprintf(sql, "INSERT INTO stb VALUES (%" PRIu64 ", %s, ?, ?)", uid, pTbCfg->name);
sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL);
pBuf = buf;
tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema);
sqlite3_bind_blob(stmt, 1, buf, POINTER_DISTANCE(pBuf, buf), NULL);
pBuf = buf;
tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema);
sqlite3_bind_blob(stmt, 2, buf, POINTER_DISTANCE(pBuf, buf), NULL);
sqlite3_step(stmt);
sqlite3_finalize(stmt);
#if 0
sprintf(sql,
"INSERT INTO tb VALUES (?, ?);"
// "INSERT INTO stb VALUES (?, ?, ?, ?);"
// "CREATE TABLE IF NOT EXISTS stb_%" PRIu64
// " ("
// " tb_uid INTEGER NOT NULL UNIQUE,"
// " tbname VARCHAR(256),"
// " tag1 INTEGER);"
,
uid);
rc = sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL);
if (rc != SQLITE_OK) {
return -1;
}
sqlite3_bind_text(stmt, 1, pTbCfg->name, -1, SQLITE_TRANSIENT);
sqlite3_bind_int64(stmt, 2, uid);
sqlite3_step(stmt);
sqlite3_finalize(stmt);
// sqlite3_bind_int64(stmt, 3, uid);
// sqlite3_bind_text(stmt, 4, pTbCfg->name, -1, SQLITE_TRANSIENT);
// pBuf = buf;
// tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema);
// sqlite3_bind_blob(stmt, 5, buf, POINTER_DISTANCE(pBuf, buf), NULL);
// pBuf = buf;
// tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema);
// sqlite3_bind_blob(stmt, 6, buf, POINTER_DISTANCE(pBuf, buf), NULL);
rc = sqliteVjj3_step(stmt);
if (rc != SQLITE_OK) {
printf("failed to create normal table since %s\n", sqlite3_errmsg(pMeta->pDB->pDB));
}
sqlite3_finalize(stmt);
#endif
break;
case META_NORMAL_TABLE:
// uid = metaGenerateUid(pMeta);
// sprintf(sql,
// "INSERT INTO tb VALUES (\'%s\', %" PRIu64
// ");"
// "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );",
// pTbCfg->name, uid, uid, pTbCfg->name, );
// rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err);
// if (rc != SQLITE_OK) {
// printf("failed to create normal table since %s\n", err);
// }
break;
case META_CHILD_TABLE:
#if 0
uid = metaGenerateUid(pMeta);
// sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64
// ");"
// "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );");
rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err);
if (rc != SQLITE_OK) {
printf("failed to create child table since %s\n", err);
}
#endif
break;
default:
break;
}
tfree(err);
return 0;
}
int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
/* TODO */
return 0;
}
\ No newline at end of file
source/dnode/vnode/meta/test/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ de8b6e73
# add_executable(metaTest "")
# target_sources(metaTest
# PRIVATE
# "../src/metaMain.c"
# "../src/metaUid.c"
# "metaTests.cpp"
# )
# target_include_directories(metaTest
# PUBLIC
# "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
# "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
# )
# target_link_libraries(metaTest
# os
# util
# common
# gtest_main
# tkv
# )
# enable_testing()
# add_test(
# NAME meta_test
# COMMAND metaTest
# )
source/dnode/vnode/meta/test/metaAPITest.cpp 已删除 100644 → 0
浏览文件 @ de8b6e73
#if 0
#include <gtest/gtest.h>
#include <string.h>
#include <iostream>
#include "meta.h"
static STSchema *metaGetSimpleSchema() {
STSchema * pSchema = NULL;
STSchemaBuilder sb = {0};
tdInitTSchemaBuilder(&sb, 0);
tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 8);
tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, 1, 4);
pSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
return pSchema;
}
static SKVRow metaGetSimpleTags() {
SKVRowBuilder kvrb = {0};
SKVRow row;
tdInitKVRowBuilder(&kvrb);
int64_t ts = 1634287978000;
int32_t a = 10;
tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_TIMESTAMP, (void *)(&ts));
tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_INT, (void *)(&a));
row = tdGetKVRowFromBuilder(&kvrb);
tdDestroyKVRowBuilder(&kvrb);
return row;
}
TEST(MetaTest, DISABLED_meta_create_1m_normal_tables_test) {
// Open Meta
SMeta *meta = metaOpen(NULL, NULL);
std::cout << "Meta is opened!" << std::endl;
// Create 1000000 normal tables
META_TABLE_OPTS_DECLARE(tbOpts);
STSchema *pSchema = metaGetSimpleSchema();
char tbname[128];
for (size_t i = 0; i < 1000000; i++) {
sprintf(tbname, "ntb%ld", i);
metaNormalTableOptsInit(&tbOpts, tbname, pSchema);
metaCreateTable(meta, &tbOpts);
metaTableOptsClear(&tbOpts);
}
tdFreeSchema(pSchema);
// Close Meta
metaClose(meta);
std::cout << "Meta is closed!" << std::endl;
// Destroy Meta
metaDestroy("meta");
std::cout << "Meta is destroyed!" << std::endl;
}
TEST(MetaTest, meta_create_1m_child_tables_test) {
// Open Meta
SMeta *meta = metaOpen(NULL);
std::cout << "Meta is opened!" << std::endl;
// Create a super tables
tb_uid_t uid = 477529885843758ul;
META_TABLE_OPTS_DECLARE(tbOpts);
STSchema *pSchema = metaGetSimpleSchema();
STSchema *pTagSchema = metaGetSimpleSchema();
metaSuperTableOptsInit(&tbOpts, "st", uid, pSchema, pTagSchema);
metaCreateTable(meta, &tbOpts);
metaTableOptsClear(&tbOpts);
tdFreeSchema(pSchema);
tdFreeSchema(pTagSchema);
// Create 1000000 child tables
char name[128];
SKVRow row = metaGetSimpleTags();
for (size_t i = 0; i < 1000000; i++) {
sprintf(name, "ctb%ld", i);
metaChildTableOptsInit(&tbOpts, name, uid, row);
metaCreateTable(meta, &tbOpts);
metaTableOptsClear(&tbOpts);
}
kvRowFree(row);
// Close Meta
metaClose(meta);
std::cout << "Meta is closed!" << std::endl;
// Destroy Meta
metaDestroy("meta");
std::cout << "Meta is destroyed!" << std::endl;
}
#endif
\ No newline at end of file
source/dnode/vnode/impl/inc/vnodeDef.h → source/dnode/vnode/src/inc/vnodeDef.h
浏览文件 @ ae2d77e9
......@@ -17,7 +17,7 @@
#define _TD_VNODE_DEF_H_
#include "mallocator.h"
#include "sync.h"
// #include "sync.h"
#include "tcoding.h"
#include "tlist.h"
#include "tlockfree.h"
......@@ -30,12 +30,9 @@
#include "vnodeBufferPool.h"
#include "vnodeCfg.h"
#include "vnodeCommit.h"
#include "vnodeFS.h"
#include "vnodeMemAllocator.h"
#include "vnodeQuery.h"
#include "vnodeRequest.h"
#include "vnodeStateMgr.h"
#include "vnodeSync.h"
#ifdef __cplusplus
extern "C" {
......@@ -73,8 +70,6 @@ struct SVnode {
STsdb* pTsdb;
STQ* pTq;
SWal* pWal;
SVnodeSync* pSync;
SVnodeFS* pFs;
tsem_t canCommit;
SQHandle* pQuery;
SDnode* pDnode;
......@@ -84,6 +79,16 @@ int vnodeScheduleTask(SVnodeTask* task);
int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq);
// For Log
extern int32_t vDebugFlag;
#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }} while(0)
#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }} while(0)
#define vWarn(...) do { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }} while(0)
#define vInfo(...) do { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("VND ", 255, __VA_ARGS__); }} while(0)
#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}
#endif
......
source/dnode/vnode/impl/inc/vnodeInt.h → source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @ ae2d77e9
......@@ -19,7 +19,7 @@
#include "vnode.h"
#include "meta.h"
#include "sync.h"
// #include "sync.h"
#include "tlog.h"
#include "tq.h"
#include "tsdb.h"
......
source/dnode/vnode/impl/inc/vnodeQuery.h → source/dnode/vnode/src/inc/vnodeQuery.h
浏览文件 @ ae2d77e9
......@@ -19,11 +19,11 @@
#ifdef __cplusplus
extern "C" {
#endif
#include "vnodeInt.h"
#include "qworker.h"
#include "vnode.h"
typedef struct SQWorkerMgmt SQHandle;
typedef struct SQWorkerMgmt SQHandle;
int vnodeQueryOpen(SVnode *pVnode);
......
source/dnode/vnode/meta/src/metaIdx.c → source/dnode/vnode/src/meta/metaIdx.c
浏览文件 @ ae2d77e9
......@@ -13,7 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef USE_INVERTED_INDEX
#include "index.h"
#endif
#include "metaDef.h"
struct SMetaIdx {
......
source/dnode/vnode/src/tsdb/tsdbHealth.c 0 → 100644
浏览文件 @ ae2d77e9
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// #include "os.h"
// #include "tmsg.h"
// #include "tarray.h"
// #include "query.h"
// #include "tglobal.h"
// #include "tlist.h"
// #include "tsdbint.h"
// #include "tsdbBuffer.h"
// #include "tsdbLog.h"
// #include "tsdbHealth.h"
// #include "ttimer.h"
// #include "tthread.h"
// // return malloc new block count
// int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) {
// STsdbBufPool *pPool = pRepo->pPool;
// int32_t cnt = 0;
// if(tsdbAllowNewBlock(pRepo)) {
// STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
// if (pBufBlock) {
// if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
// // append error
// tsdbFreeBufBlock(pBufBlock);
// } else {
// pPool->nElasticBlocks ++;
// cnt ++ ;
// }
// }
// }
// return cnt;
// }
// // switch anther thread to run
// void* cbKillQueryFree(void* param) {
// STsdbRepo* pRepo = (STsdbRepo*)param;
// // vnode
// if(pRepo->appH.notifyStatus) {
// pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS);
// }
// // free
// if(pRepo->pthread){
// void* p = pRepo->pthread;
// pRepo->pthread = NULL;
// free(p);
// }
// return NULL;
// }
// // return true do free , false do nothing
// bool tsdbUrgeQueryFree(STsdbRepo * pRepo) {
// // check previous running
// if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) {
// tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks);
// return false;
// }
// // create new
// pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo);
// if(pRepo->pthread == NULL) {
// tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo));
// return false;
// }
// return true;
// }
// bool tsdbAllowNewBlock(STsdbRepo* pRepo) {
// int32_t nMaxElastic = pRepo->config.totalBlocks/3;
// STsdbBufPool* pPool = pRepo->pPool;
// if(pPool->nElasticBlocks >= nMaxElastic) {
// tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic);
// return false;
// }
// return true;
// }
// bool tsdbNoProblem(STsdbRepo* pRepo) {
// if(listNEles(pRepo->pPool->bufBlockList) == 0)
// return false;
// return true;
// }
\ No newline at end of file
source/dnode/vnode/tsdb/src/tsdbRead.c → source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ ae2d77e9
......@@ -28,7 +28,7 @@
#include "taosdef.h"
#include "tlosertree.h"
#include "tsdbint.h"
#include "tsdbDef.h"
#include "tmsg.h"
#define EXTRA_BYTES 2
......@@ -713,13 +713,6 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
pCheckInfo->initBuf = true;
int32_t order = pHandle->order;
// no data in buffer, abort
// if (pHandle->pMemTable->snapshot.mem == NULL && pHandle->pMemTable->snapshot.imem == NULL) {
// return false;
// }
//
// assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL);
//
STbData** pMem = NULL;
STbData** pIMem = NULL;
......@@ -787,8 +780,7 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
assert(pCheckInfo->lastKey >= key);
}
} else {
tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId,
pHandle->qId);
tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId);
}
return true;
......@@ -2554,9 +2546,6 @@ static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) {
pTsdbReadHandle->activeIndex += 1;
}
// no data in memtable or imemtable, decrease the memory reference.
// TODO !!
// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
return false;
}
......
source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c → source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c
浏览文件 @ ae2d77e9
......@@ -13,18 +13,18 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdbRowMergeBuf.h"
#include "tdataformat.h"
// #include "tsdbRowMergeBuf.h"
// #include "tdataformat.h"
// row1 has higher priority
SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) {
if(row2 == NULL) return row1;
if(row1 == NULL) return row2;
ASSERT(pSchema1->version == memRowVersion(row1));
ASSERT(pSchema2->version == memRowVersion(row2));
// // row1 has higher priority
// SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) {
// if(row2 == NULL) return row1;
// if(row1 == NULL) return row2;
// ASSERT(pSchema1->version == memRowVersion(row1));
// ASSERT(pSchema2->version == memRowVersion(row2));
if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) {
return NULL;
}
return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2);
}
// if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) {
// return NULL;
// }
// return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2);
// }
source/dnode/vnode/tsdb/src/tsdbScan.c → source/dnode/vnode/src/tsdb/tsdbScan.c
浏览文件 @ ae2d77e9
......@@ -13,9 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdbint.h"
#if 0
#include "tsdbint.h"
#ifndef _TSDB_PLUGINS
int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; }
......
source/dnode/vnode/tsdb/src/tsdbWrite.c → source/dnode/vnode/src/tsdb/tsdbWrite.c
浏览文件 @ ae2d77e9
......@@ -17,9 +17,11 @@
int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp) {
// Check if mem is there. If not, create one.
if (pTsdb->mem == NULL) {
pTsdb->mem = tsdbNewMemTable(pTsdb);
if (pTsdb->mem == NULL) {
return -1;
}
}
return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL);
}
\ No newline at end of file
source/dnode/vnode/impl/src/vnodeMgr.c → source/dnode/vnode/src/vnd/vnodeMgr.c
浏览文件 @ ae2d77e9
......@@ -41,7 +41,7 @@ int vnodeInit(const SVnodeOpt *pOption) {
for (uint16_t i = 0; i < pOption->nthreads; i++) {
pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL);
pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread");
// pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread");
}
} else {
// TODO: if no commit thread is set, then another mechanism should be
......
source/dnode/vnode/impl/src/vnodeQuery.c → source/dnode/vnode/src/vnd/vnodeQuery.c
浏览文件 @ ae2d77e9
......@@ -19,11 +19,22 @@
static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg);
static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); }
int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, &pVnode->pQuery, pVnode, vnodePutReqToVQueryQ); }
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vTrace("query message is processed");
vTrace("query message is processing");
switch (pMsg->msgType) {
case TDMT_VND_QUERY:
return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg);
case TDMT_VND_QUERY_CONTINUE:
return qWorkerProcessQueryContinueMsg(pVnode->pTsdb, pVnode->pQuery, pMsg);
case TDMT_VND_SCHEDULE_DATA_SINK:
return qWorkerProcessDataSinkMsg(pVnode->pTsdb, pVnode->pQuery, pMsg);
default:
vError("unknown msg type:%d in query queue", pMsg->msgType);
return TSDB_CODE_VND_APP_ERROR;
}
}
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
......@@ -68,6 +79,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid);
if (pTbCfg == NULL) {
code = TSDB_CODE_VND_TB_NOT_EXIST;
goto _exit;
}
......
source/dnode/vnode/impl/src/vnodeWrite.c → source/dnode/vnode/src/vnd/vnodeWrite.c
浏览文件 @ ae2d77e9
......@@ -83,7 +83,18 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) {
// TODO: handle error
}
vTrace("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name);
if (pCreateTbReq->type == TD_SUPER_TABLE) {
free(pCreateTbReq->stbCfg.pSchema);
free(pCreateTbReq->stbCfg.pTagSchema);
} else if (pCreateTbReq->type == TD_CHILD_TABLE) {
free(pCreateTbReq->ctbCfg.pTag);
} else {
free(pCreateTbReq->ntbCfg.pSchema);
}
}
taosArrayDestroy(vCreateTbBatchReq.pArray);
break;
case TDMT_VND_DROP_STB:
case TDMT_VND_DROP_TABLE:
......@@ -105,7 +116,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
// Check if it needs to commit
if (vnodeShouldCommit(pVnode)) {
tsem_wait(&(pVnode->canCommit));
// tsem_wait(&(pVnode->canCommit));
if (vnodeAsyncCommit(pVnode) < 0) {
// TODO: handle error
}
......
source/dnode/vnode/tsdb/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ de8b6e73
aux_source_directory(src TSDB_SRC)
if(0)
add_library(tsdb ${TSDB_SRC})
else(0)
add_library(tsdb STATIC "")
target_sources(tsdb
PRIVATE
"src/tsdbCommit.c"
"src/tsdbMain.c"
"src/tsdbMemTable.c"
"src/tsdbOptions.c"
"src/tsdbWrite.c"
"src/tsdbReadImpl.c"
"src/tsdbFile.c"
"src/tsdbFS.c"
"src/tsdbRead.c"
)
endif(0)
target_include_directories(
tsdb
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tsdb"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
tsdb
PUBLIC os
PUBLIC util
PUBLIC common
PUBLIC tkv
PUBLIC tfs
PUBLIC meta
)
\ No newline at end of file
source/dnode/vnode/tsdb/inc/tsdbint.h 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TSDB_INT_H_
#define _TD_TSDB_INT_H_
#if 0
// // TODO: remove the include
// #include <errno.h>
// #include <fcntl.h>
// #include <limits.h>
// #include <inttypes.h>
// #include <sys/stat.h>
// #include <sys/types.h>
// #include <semaphore.h>
// #include <dirent.h>
#include "hash.h"
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tarray.h"
#include "tchecksum.h"
#include "tcoding.h"
#include "tcompression.h"
#include "tdataformat.h"
#include "tfs.h"
#include "tlist.h"
#include "tlockfree.h"
#include "tlog.h"
#include "tskiplist.h"
#include "tsocket.h"
#include "tsdb.h"
#ifdef __cplusplus
extern "C" {
#endif
// Log
#include "tsdbLog.h"
// Meta
#include "tsdbMeta.h"
// Buffer
#include "tsdbBuffer.h"
// MemTable
#include "tsdbMemTable.h"
// File
#include "tsdbFile.h"
// FS
#include "tsdbFS.h"
// ReadImpl
#include "tsdbReadImpl.h"
// Commit
#include "tsdbCommit.h"
// Compact
#include "tsdbCompact.h"
// Commit Queue
#include "tsdbCommitQueue.h"
#include "tsdbRowMergeBuf.h"
// Main definitions
struct STsdbRepo {
uint8_t state;
STsdbCfg config;
STsdbCfg save_config; // save apply config
bool config_changed; // config changed flag
pthread_mutex_t save_mutex; // protect save config
uint8_t hasCachedLastColumn;
STsdbAppH appH;
STsdbStat stat;
STsdbMeta* tsdbMeta;
STsdbBufPool* pPool;
SMemTable* mem;
SMemTable* imem;
STsdbFS* fs;
SRtn rtn;
tsem_t readyToCommit;
pthread_mutex_t mutex;
bool repoLocked;
int32_t code; // Commit code
SMergeBuf mergeBuf; //used when update=2
int8_t compactState; // compact state: inCompact/noCompact/waitingCompact?
pthread_t* pthread;
};
#define REPO_ID(r) (r)->config.tsdbId
#define REPO_CFG(r) (&((r)->config))
#define REPO_FS(r) ((r)->fs)
#define IS_REPO_LOCKED(r) (r)->repoLocked
#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
int tsdbLockRepo(STsdbRepo* pRepo);
int tsdbUnlockRepo(STsdbRepo* pRepo);
STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo);
int tsdbCheckCommit(STsdbRepo* pRepo);
int tsdbRestoreInfo(STsdbRepo* pRepo);
int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg);
void tsdbGetRootDir(int repoid, char dirName[]);
void tsdbGetDataDir(int repoid, char dirName[]);
static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
ASSERT(pRepo != NULL);
if (pRepo->mem == NULL) return NULL;
SListNode* pNode = listTail(pRepo->mem->bufBlockList);
if (pNode == NULL) return NULL;
STsdbBufBlock* pBufBlock = NULL;
tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock));
return pBufBlock;
}
static FORCE_INLINE int tsdbGetNextMaxTables(int tid) {
ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES);
int maxTables = TSDB_INIT_NTABLES;
while (true) {
maxTables = MIN(maxTables, TSDB_MAX_TABLES);
if (tid <= maxTables) break;
maxTables *= 2;
}
return maxTables + 1;
}
#ifdef __cplusplus
}
#endif
#endif
#endif /* _TD_TSDB_INT_H_ */
source/dnode/vnode/tsdb/src/tsdbHealth.c 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tmsg.h"
#include "tarray.h"
#include "query.h"
#include "tglobal.h"
#include "tlist.h"
#include "tsdbint.h"
#include "tsdbBuffer.h"
#include "tsdbLog.h"
#include "tsdbHealth.h"
#include "ttimer.h"
#include "tthread.h"
// return malloc new block count
int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) {
STsdbBufPool *pPool = pRepo->pPool;
int32_t cnt = 0;
if(tsdbAllowNewBlock(pRepo)) {
STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
if (pBufBlock) {
if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
// append error
tsdbFreeBufBlock(pBufBlock);
} else {
pPool->nElasticBlocks ++;
cnt ++ ;
}
}
}
return cnt;
}
// switch anther thread to run
void* cbKillQueryFree(void* param) {
STsdbRepo* pRepo = (STsdbRepo*)param;
// vnode
if(pRepo->appH.notifyStatus) {
pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS);
}
// free
if(pRepo->pthread){
void* p = pRepo->pthread;
pRepo->pthread = NULL;
free(p);
}
return NULL;
}
// return true do free , false do nothing
bool tsdbUrgeQueryFree(STsdbRepo * pRepo) {
// check previous running
if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) {
tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks);
return false;
}
// create new
pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo);
if(pRepo->pthread == NULL) {
tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo));
return false;
}
return true;
}
bool tsdbAllowNewBlock(STsdbRepo* pRepo) {
int32_t nMaxElastic = pRepo->config.totalBlocks/3;
STsdbBufPool* pPool = pRepo->pPool;
if(pPool->nElasticBlocks >= nMaxElastic) {
tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic);
return false;
}
return true;
}
bool tsdbNoProblem(STsdbRepo* pRepo) {
if(listNEles(pRepo->pPool->bufBlockList) == 0)
return false;
return true;
}
\ No newline at end of file
source/dnode/vnode/tsdb/src/tsdbSync.c 已删除 100644 → 0
浏览文件 @ de8b6e73
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tsdbint.h"
// Sync handle
typedef struct {
STsdbRepo *pRepo;
SRtn rtn;
SOCKET socketFd;
void * pBuf;
bool mfChanged;
SMFile * pmf;
SMFile mf;
SDFileSet df;
SDFileSet *pdf;
} SSyncH;
#define SYNC_BUFFER(sh) ((sh)->pBuf)
static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd);
static void tsdbDestroySyncH(SSyncH *pSyncH);
static int32_t tsdbSyncSendMeta(SSyncH *pSynch);
static int32_t tsdbSyncRecvMeta(SSyncH *pSynch);
static int32_t tsdbSendMetaInfo(SSyncH *pSynch);
static int32_t tsdbRecvMetaInfo(SSyncH *pSynch);
static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend);
static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend);
static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch);
static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch);
static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2);
static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet);
static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet);
static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch);
static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged);
int32_t tsdbSyncSend(void *tsdb, SOCKET socketFd) {
STsdbRepo *pRepo = (STsdbRepo *)tsdb;
SSyncH synch = {0};
tsdbInitSyncH(&synch, pRepo, socketFd);
// Disable TSDB commit
tsem_wait(&(pRepo->readyToCommit));
if (tsdbSyncSendMeta(&synch) < 0) {
tsdbError("vgId:%d, failed to send metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
if (tsdbSyncSendDFileSetArray(&synch) < 0) {
tsdbError("vgId:%d, failed to send filesets since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
// Enable TSDB commit
tsem_post(&(pRepo->readyToCommit));
tsdbDestroySyncH(&synch);
return 0;
_err:
tsem_post(&(pRepo->readyToCommit));
tsdbDestroySyncH(&synch);
return -1;
}
int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) {
STsdbRepo *pRepo = (STsdbRepo *)tsdb;
SSyncH synch = {0};
pRepo->state = TSDB_STATE_OK;
tsdbInitSyncH(&synch, pRepo, socketFd);
tsem_wait(&(pRepo->readyToCommit));
tsdbStartFSTxn(pRepo, 0, 0);
if (tsdbSyncRecvMeta(&synch) < 0) {
tsdbError("vgId:%d, failed to recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
if (tsdbSyncRecvDFileSetArray(&synch) < 0) {
tsdbError("vgId:%d, failed to recv filesets since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
tsdbEndFSTxn(pRepo);
tsem_post(&(pRepo->readyToCommit));
tsdbDestroySyncH(&synch);
// Reload file change
tsdbReload(pRepo, synch.mfChanged);
return 0;
_err:
tsdbEndFSTxnWithError(REPO_FS(pRepo));
tsem_post(&(pRepo->readyToCommit));
tsdbDestroySyncH(&synch);
return -1;
}
static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd) {
pSyncH->pRepo = pRepo;
pSyncH->socketFd = socketFd;
tsdbGetRtnSnap(pRepo, &(pSyncH->rtn));
}
static void tsdbDestroySyncH(SSyncH *pSyncH) { taosTZfree(pSyncH->pBuf); }
static int32_t tsdbSyncSendMeta(SSyncH *pSynch) {
STsdbRepo *pRepo = pSynch->pRepo;
bool toSendMeta = false;
SMFile mf;
// Send meta info to remote
tsdbInfo("vgId:%d, metainfo will be sent", REPO_ID(pRepo));
if (tsdbSendMetaInfo(pSynch) < 0) {
tsdbError("vgId:%d, failed to send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
if (pRepo->fs->cstatus->pmf == NULL) {
// No meta file, not need to wait to retrieve meta file
tsdbInfo("vgId:%d, metafile not exist, no need to send", REPO_ID(pRepo));
return 0;
}
if (tsdbRecvDecision(pSynch, &toSendMeta) < 0) {
tsdbError("vgId:%d, failed to recv decision while send meta since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
if (toSendMeta) {
tsdbInitMFileEx(&mf, pRepo->fs->cstatus->pmf);
if (tsdbOpenMFile(&mf, O_RDONLY) < 0) {
tsdbError("vgId:%d, failed to open file while send metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
int64_t writeLen = mf.info.size;
tsdbInfo("vgId:%d, metafile:%s will be sent, size:%" PRId64, REPO_ID(pRepo), mf.f.aname, writeLen);
int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&mf), 0, writeLen);
if (ret != writeLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to send metafile since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo),
tstrerror(terrno), ret, writeLen);
tsdbCloseMFile(&mf);
return -1;
}
tsdbCloseMFile(&mf);
tsdbInfo("vgId:%d, metafile is sent", REPO_ID(pRepo));
} else {
tsdbInfo("vgId:%d, metafile is same, no need to send", REPO_ID(pRepo));
}
return 0;
}
static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) {
STsdbRepo *pRepo = pSynch->pRepo;
SMFile * pLMFile = pRepo->fs->cstatus->pmf;
// Recv meta info from remote
if (tsdbRecvMetaInfo(pSynch) < 0) {
tsdbError("vgId:%d, failed to recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
// No meta file, do nothing (rm local meta file)
if (pSynch->pmf == NULL) {
if (pLMFile == NULL) {
pSynch->mfChanged = false;
} else {
pSynch->mfChanged = true;
}
tsdbInfo("vgId:%d, metafile not exist in remote, no need to recv", REPO_ID(pRepo));
return 0;
}
if (pLMFile == NULL || pSynch->pmf->info.size != pLMFile->info.size ||
pSynch->pmf->info.magic != pLMFile->info.magic || TSDB_FILE_IS_BAD(pLMFile)) {
// Local has no meta file or has a different meta file, need to copy from remote
pSynch->mfChanged = true;
if (tsdbSendDecision(pSynch, true) < 0) {
tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
tsdbInfo("vgId:%d, metafile will be received", REPO_ID(pRepo));
// Recv from remote
SMFile mf;
SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID};
tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
if (tsdbCreateMFile(&mf, false) < 0) {
tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
tsdbInfo("vgId:%d, metafile:%s is created", REPO_ID(pRepo), mf.f.aname);
int64_t readLen = pSynch->pmf->info.size;
int64_t ret = taosCopyFds(pSynch->socketFd, TSDB_FILE_FD(&mf), readLen);
if (ret != readLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to recv metafile since %s, ret:%" PRId64 " readLen:%" PRId64, REPO_ID(pRepo),
tstrerror(terrno), ret, readLen);
tsdbCloseMFile(&mf);
tsdbRemoveMFile(&mf);
return -1;
}
tsdbInfo("vgId:%d, metafile is received, size:%" PRId64, REPO_ID(pRepo), readLen);
mf.info = pSynch->pmf->info;
tsdbCloseMFile(&mf);
tsdbUpdateMFile(REPO_FS(pRepo), &mf);
} else {
pSynch->mfChanged = false;
tsdbInfo("vgId:%d, metafile is same, no need to recv", REPO_ID(pRepo));
if (tsdbSendDecision(pSynch, false) < 0) {
tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
tsdbUpdateMFile(REPO_FS(pRepo), pLMFile);
}
return 0;
}
static int32_t tsdbSendMetaInfo(SSyncH *pSynch) {
STsdbRepo *pRepo = pSynch->pRepo;
uint32_t tlen = 0;
SMFile * pMFile = pRepo->fs->cstatus->pmf;
if (pMFile) {
tlen = tlen + tsdbEncodeSMFileEx(NULL, pMFile) + sizeof(TSCKSUM);
}
if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) {
tsdbError("vgId:%d, failed to makeroom while send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
void *ptr = SYNC_BUFFER(pSynch);
taosEncodeFixedU32(&ptr, tlen);
void *tptr = ptr;
if (pMFile) {
tsdbEncodeSMFileEx(&ptr, pMFile);
taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen);
}
int32_t writeLen = tlen + sizeof(uint32_t);
int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen);
if (ret != writeLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to send metainfo since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret,
writeLen);
return -1;
}
tsdbInfo("vgId:%d, metainfo is sent, tlen:%d, writeLen:%d", REPO_ID(pRepo), tlen, writeLen);
return 0;
}
static int32_t tsdbRecvMetaInfo(SSyncH *pSynch) {
STsdbRepo *pRepo = pSynch->pRepo;
uint32_t tlen = 0;
char buf[64] = {0};
int32_t readLen = sizeof(uint32_t);
int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen);
if (ret != readLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to recv metalen, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen);
return -1;
}
taosDecodeFixedU32(buf, &tlen);
tsdbInfo("vgId:%d, metalen is received, readLen:%d, tlen:%d", REPO_ID(pRepo), readLen, tlen);
if (tlen == 0) {
pSynch->pmf = NULL;
return 0;
}
if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) {
tsdbError("vgId:%d, failed to makeroom while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen);
if (ret != tlen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to recv metainfo, ret:%d tlen:%d", REPO_ID(pRepo), ret, tlen);
return -1;
}
tsdbInfo("vgId:%d, metainfo is received, tlen:%d", REPO_ID(pRepo), tlen);
if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) {
terrno = TSDB_CODE_TDB_MESSED_MSG;
tsdbError("vgId:%d, failed to checksum while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
pSynch->pmf = &(pSynch->mf);
tsdbDecodeSMFileEx(SYNC_BUFFER(pSynch), pSynch->pmf);
return 0;
}
static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend) {
STsdbRepo *pRepo = pSynch->pRepo;
uint8_t decision = toSend;
int32_t writeLen = sizeof(uint8_t);
int32_t ret = taosWriteMsg(pSynch->socketFd, (void *)(&decision), writeLen);
if (ret != writeLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to send decison, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen);
return -1;
}
return 0;
}
static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend) {
STsdbRepo *pRepo = pSynch->pRepo;
uint8_t decision = 0;
int32_t readLen = sizeof(uint8_t);
int32_t ret = taosReadMsg(pSynch->socketFd, (void *)(&decision), readLen);
if (ret != readLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to recv decison, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen);
return -1;
}
*toSend = decision;
return 0;
}
static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch) {
STsdbRepo *pRepo = pSynch->pRepo;
STsdbFS * pfs = REPO_FS(pRepo);
SFSIter fsiter;
SDFileSet *pSet;
tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
do {
pSet = tsdbFSIterNext(&fsiter);
if (tsdbSyncSendDFileSet(pSynch, pSet) < 0) {
tsdbError("vgId:%d, failed to send fileset:%d since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1,
tstrerror(terrno));
return -1;
}
// No more file set to send, jut break
if (pSet == NULL) {
tsdbInfo("vgId:%d, no filesets any more", REPO_ID(pRepo));
break;
}
} while (true);
return 0;
}
static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) {
STsdbRepo *pRepo = pSynch->pRepo;
STsdbFS * pfs = REPO_FS(pRepo);
SFSIter fsiter;
SDFileSet *pLSet; // Local file set
tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
pLSet = tsdbFSIterNext(&fsiter);
if (tsdbRecvDFileSetInfo(pSynch) < 0) {
tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
while (true) {
if (pLSet == NULL && pSynch->pdf == NULL) {
tsdbInfo("vgId:%d, all filesets is disposed", REPO_ID(pRepo));
break;
} else {
tsdbInfo("vgId:%d, fileset local:%d remote:%d, will be disposed", REPO_ID(pRepo), pLSet != NULL ? pLSet->fid : -1,
pSynch->pdf != NULL ? pSynch->pdf->fid : -1);
}
if (pLSet && (pSynch->pdf == NULL || pLSet->fid < pSynch->pdf->fid)) {
// remote not has pLSet->fid set, just remove local (do nothing to remote the fset)
tsdbInfo("vgId:%d, fileset:%d smaller than remote:%d, remove it", REPO_ID(pRepo), pLSet->fid,
pSynch->pdf != NULL ? pSynch->pdf->fid : -1);
pLSet = tsdbFSIterNext(&fsiter);
} else {
if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf) &&
tsdbFSetIsOk(pLSet)) {
// Just keep local files and notify remote not to send
tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid);
if (tsdbUpdateDFileSet(pfs, pLSet) < 0) {
tsdbError("vgId:%d, failed to update fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
if (tsdbSendDecision(pSynch, false) < 0) {
tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
} else {
// Need to copy from remote
int fidLevel = tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn));
if (fidLevel < 0) { // expired fileset
tsdbInfo("vgId:%d, fileset:%d will be skipped as expired", REPO_ID(pRepo), pSynch->pdf->fid);
if (tsdbSendDecision(pSynch, false) < 0) {
tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
// Move forward
if (tsdbRecvDFileSetInfo(pSynch) < 0) {
tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
if (pLSet) {
pLSet = tsdbFSIterNext(&fsiter);
}
// Next loop
continue;
} else {
tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid);
// Notify remote to send there file here
if (tsdbSendDecision(pSynch, true) < 0) {
tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
}
// Create local files and copy from remote
SDiskID did;
SDFileSet fset;
tfsAllocDisk(fidLevel, &(did.level), &(did.id));
if (did.level == TFS_UNDECIDED_LEVEL) {
terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
tsdbInitDFileSet(&fset, did, REPO_ID(pRepo), pSynch->pdf->fid, FS_TXN_VERSION(pfs));
// Create new FSET
if (tsdbCreateDFileSet(&fset, false) < 0) {
tsdbError("vgId:%d, failed to create fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); // local file
SDFile *pRDFile = TSDB_DFILE_IN_SET(pSynch->pdf, ftype); // remote file
tsdbInfo("vgId:%d, file:%s will be received, osize:%" PRIu64 " rsize:%" PRIu64, REPO_ID(pRepo),
pDFile->f.aname, pDFile->info.size, pRDFile->info.size);
int64_t writeLen = pRDFile->info.size;
int64_t ret = taosCopyFds(pSynch->socketFd, pDFile->fd, writeLen);
if (ret != writeLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to recv file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo),
pDFile->f.aname, tstrerror(terrno), ret, writeLen);
tsdbCloseDFileSet(&fset);
tsdbRemoveDFileSet(&fset);
return -1;
}
// Update new file info
pDFile->info = pRDFile->info;
tsdbInfo("vgId:%d, file:%s is received, size:%" PRId64, REPO_ID(pRepo), pDFile->f.aname, writeLen);
}
tsdbCloseDFileSet(&fset);
if (tsdbUpdateDFileSet(pfs, &fset) < 0) {
tsdbInfo("vgId:%d, fileset:%d failed to update since %s", REPO_ID(pRepo), fset.fid, tstrerror(terrno));
return -1;
}
tsdbInfo("vgId:%d, fileset:%d is received", REPO_ID(pRepo), pSynch->pdf->fid);
}
// Move forward
if (tsdbRecvDFileSetInfo(pSynch) < 0) {
tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
if (pLSet) {
pLSet = tsdbFSIterNext(&fsiter);
}
}
#if 0
if (pLSet == NULL) {
// Copy from remote >>>>>>>>>>>
} else {
if (pSynch->pdf == NULL) {
// Remove local file, just ignore ++++++++++++++
pLSet = tsdbFSIterNext(&fsiter);
} else {
if (pLSet->fid < pSynch->pdf->fid) {
// Remove local file, just ignore ++++++++++++
pLSet = tsdbFSIterNext(&fsiter);
} else if (pLSet->fid > pSynch->pdf->fid){
// Copy from remote >>>>>>>>>>>>>>
if (tsdbRecvDFileSetInfo(pSynch) < 0) {
// TODO
return -1;
}
} else {
if (true/*TODO: is same fset*/) {
// No need to copy ---------------------
} else {
// copy from remote >>>>>>>>>>>>>.
}
}
}
}
#endif
}
return 0;
}
static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype);
SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype);
if (pDFile1->info.size != pDFile2->info.size || pDFile1->info.magic != pDFile2->info.magic) {
return false;
}
}
return true;
}
static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) {
STsdbRepo *pRepo = pSynch->pRepo;
bool toSend = false;
// skip expired fileset
if (pSet && tsdbGetFidLevel(pSet->fid, &(pSynch->rtn)) < 0) {
tsdbInfo("vgId:%d, don't sync send since fileset:%d smaller than minFid:%d", REPO_ID(pRepo), pSet->fid,
pSynch->rtn.minFid);
return 0;
}
if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) {
tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, tstrerror(terrno));
return -1;
}
// No file any more, no need to send file, just return
if (pSet == NULL) {
return 0;
}
if (tsdbRecvDecision(pSynch, &toSend) < 0) {
tsdbError("vgId:%d, failed to recv decision while send fileset:%d since %s", REPO_ID(pRepo), pSet->fid,
tstrerror(terrno));
return -1;
}
if (toSend) {
tsdbInfo("vgId:%d, fileset:%d will be sent", REPO_ID(pRepo), pSet->fid);
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
SDFile df = *TSDB_DFILE_IN_SET(pSet, ftype);
if (tsdbOpenDFile(&df, O_RDONLY) < 0) {
tsdbError("vgId:%d, failed to file:%s since %s", REPO_ID(pRepo), df.f.aname, tstrerror(terrno));
return -1;
}
int64_t writeLen = df.info.size;
tsdbInfo("vgId:%d, file:%s will be sent, size:%" PRId64, REPO_ID(pRepo), df.f.aname, writeLen);
int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&df), 0, writeLen);
if (ret != writeLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to send file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo),
df.f.aname, tstrerror(terrno), ret, writeLen);
tsdbCloseDFile(&df);
return -1;
}
tsdbInfo("vgId:%d, file:%s is sent", REPO_ID(pRepo), df.f.aname);
tsdbCloseDFile(&df);
}
tsdbInfo("vgId:%d, fileset:%d is sent", REPO_ID(pRepo), pSet->fid);
} else {
tsdbInfo("vgId:%d, fileset:%d is same, no need to send", REPO_ID(pRepo), pSet->fid);
}
return 0;
}
static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet) {
STsdbRepo *pRepo = pSynch->pRepo;
uint32_t tlen = 0;
if (pSet) {
tlen = tsdbEncodeDFileSetEx(NULL, pSet) + sizeof(TSCKSUM);
}
if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) {
tsdbError("vgId:%d, failed to makeroom while send fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
void *ptr = SYNC_BUFFER(pSynch);
taosEncodeFixedU32(&ptr, tlen);
void *tptr = ptr;
if (pSet) {
tsdbEncodeDFileSetEx(&ptr, pSet);
taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen);
}
int32_t writeLen = tlen + sizeof(uint32_t);
int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen);
if (ret != writeLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to send fileinfo, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen);
return -1;
}
return 0;
}
static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch) {
STsdbRepo *pRepo = pSynch->pRepo;
uint32_t tlen;
char buf[64] = {0};
int32_t readLen = sizeof(uint32_t);
int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen);
if (ret != readLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
taosDecodeFixedU32(buf, &tlen);
tsdbInfo("vgId:%d, fileinfo len:%d is received", REPO_ID(pRepo), tlen);
if (tlen == 0) {
pSynch->pdf = NULL;
return 0;
}
if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) {
tsdbError("vgId:%d, failed to makeroom while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen);
if (ret != tlen) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbError("vgId:%d, failed to recv fileinfo, ret:%d readLen:%d", REPO_ID(pRepo), ret, tlen);
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) {
terrno = TSDB_CODE_TDB_MESSED_MSG;
tsdbError("vgId:%d, failed to checksum while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
pSynch->pdf = &(pSynch->df);
tsdbDecodeDFileSetEx(SYNC_BUFFER(pSynch), pSynch->pdf);
return 0;
}
static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged) {
// TODO: may need to stop and restart stream
// if (isMfChanged) {
tsdbCloseMeta(pRepo);
tsdbFreeMeta(pRepo->tsdbMeta);
pRepo->tsdbMeta = tsdbNewMeta(REPO_CFG(pRepo));
tsdbOpenMeta(pRepo);
tsdbLoadMetaCache(pRepo, true);
// }
tsdbUnRefMemTable(pRepo, pRepo->mem);
tsdbUnRefMemTable(pRepo, pRepo->imem);
pRepo->mem = NULL;
pRepo->imem = NULL;
if (tsdbRestoreInfo(pRepo) < 0) {
tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
return 0;
}
\ No newline at end of file
source/libs/CMakeLists.txt
浏览文件 @ ae2d77e9
add_subdirectory(transport)
add_subdirectory(sync)
add_subdirectory(tkv)
add_subdirectory(tdb)
add_subdirectory(index)
add_subdirectory(wal)
add_subdirectory(parser)
......
source/libs/catalog/src/catalog.c
浏览文件 @ ae2d77e9
......@@ -285,16 +285,16 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter,
char dbFullName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFullName);
ctgDebug("try to get table meta from vnode, db:%s, tbName:%s", dbFullName, pTableName->tname);
ctgDebug("try to get table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName));
SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)pTableName->tname};
SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)tNameGetTableName(pTableName)};
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen);
if (code) {
ctgError("Build vnode tablemeta msg failed, code:%x, tbName:%s", code, pTableName->tname);
ctgError("Build vnode tablemeta msg failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName));
CTG_ERR_RET(code);
}
......@@ -313,21 +313,21 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter,
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) {
SET_META_TYPE_NONE(output->metaType);
ctgDebug("tablemeta not exist in vnode, tbName:%s", pTableName->tname);
ctgDebug("tablemeta not exist in vnode, tbName:%s", tNameGetTableName(pTableName));
return TSDB_CODE_SUCCESS;
}
ctgError("error rsp for table meta from vnode, code:%x, tbName:%s", rpcRsp.code, pTableName->tname);
ctgError("error rsp for table meta from vnode, code:%x, tbName:%s", rpcRsp.code, tNameGetTableName(pTableName));
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process vnode tablemeta rsp failed, code:%x, tbName:%s", code, pTableName->tname);
ctgError("Process vnode tablemeta rsp failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName));
CTG_ERR_RET(code);
}
ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, pTableName->tname);
ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName));
return TSDB_CODE_SUCCESS;
}
......@@ -776,7 +776,7 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con
STableMetaOutput *output = &voutput;
if (CTG_IS_STABLE(isSTable)) {
ctgDebug("will renew table meta, supposed to be stable, tbName:%s", pTableName->tname);
ctgDebug("will renew table meta, supposed to be stable, tbName:%s", tNameGetTableName(pTableName));
// if get from mnode failed, will not try vnode
CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput));
......@@ -787,13 +787,13 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con
output = &moutput;
}
} else {
ctgDebug("will renew table meta, not supposed to be stable, tbName:%s, isStable:%d", pTableName->tname, isSTable);
ctgDebug("will renew table meta, not supposed to be stable, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable);
// if get from vnode failed or no table meta, will not try mnode
CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput));
if (CTG_IS_META_TABLE(voutput.metaType) && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) {
ctgDebug("will continue to renew table meta since got stable, tbName:%s, metaType:%d", pTableName->tname, voutput.metaType);
ctgDebug("will continue to renew table meta since got stable, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType);
CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput));
......@@ -820,7 +820,7 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con
}
if (CTG_IS_META_NONE(output->metaType)) {
ctgError("no tablemeta got, tbNmae:%s", pTableName->tname);
ctgError("no tablemeta got, tbNmae:%s", tNameGetTableName(pTableName));
CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
}
......@@ -860,7 +860,7 @@ int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMg
CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist));
if (0 == exist) {
ctgError("renew tablemeta succeed but get from cache failed, may be deleted, tbName:%s", pTableName->tname);
ctgError("renew tablemeta succeed but get from cache failed, may be deleted, tbName:%s", tNameGetTableName(pTableName));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
......@@ -1241,7 +1241,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S
} else {
int32_t vgId = tbMeta->vgId;
if (NULL == taosHashGetClone(dbVgroup->vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) {
ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, pTableName->tname);
ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
......@@ -1252,7 +1252,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S
}
if (NULL == taosArrayPush(vgList, &vgroupInfo)) {
ctgError("taosArrayPush vgroupInfo to array failed, vgId:%d, tbName:%s", vgId, pTableName->tname);
ctgError("taosArrayPush vgroupInfo to array failed, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
......
source/libs/executor/CMakeLists.txt
浏览文件 @ ae2d77e9
......@@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC})
# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor"
# )
target_link_libraries(executor
PRIVATE os util common function parser planner qcom tsdb
PRIVATE os util common function parser planner qcom vnode
)
target_include_directories(
......
source/libs/executor/inc/executorimpl.h
浏览文件 @ ae2d77e9
......@@ -374,6 +374,12 @@ typedef struct STaskParam {
struct SUdfInfo *pUdfInfo;
} STaskParam;
typedef struct SExchangeInfo {
int32_t numOfSources;
SEpSet *pEpset;
int32_t bytes; // total load bytes from remote
} SExchangeInfo;
typedef struct STableScanInfo {
void *pTsdbReadHandle;
int32_t numOfBlocks;
......@@ -393,12 +399,9 @@ typedef struct STableScanInfo {
SSDataBlock block;
int32_t numOfOutput;
int64_t elapsedTime;
int32_t tableIndex;
int32_t prevGroupId; // previous table group id
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
STimeWindow window;
} STableScanInfo;
typedef struct STagScanInfo {
......@@ -542,34 +545,36 @@ typedef struct SOrderOperatorInfo {
void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime);
SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream);
SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult);
SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createExchangeOperatorInfo(const SVgroupInfo* pVgroups, int32_t numOfSources, int32_t numOfOutput, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream);
SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult);
SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
int32_t numOfRows, void* merger);
SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp);
SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult);
SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr,
SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp);
SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult);
SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput);
SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal);
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput);
SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal);
//SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
//SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
......
source/libs/executor/src/dataDispatcher.c
浏览文件 @ ae2d77e9
......@@ -182,6 +182,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
if (NULL == pDispatcher->nextOutput.pData) {
assert(pDispatcher->queryEnd);
pOutput->useconds = pDispatcher->useconds;
pOutput->precision = pDispatcher->schema.precision;
return TSDB_CODE_SUCCESS;
}
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
pOutput->numOfRows = pEntry->numOfRows;
......
source/libs/executor/src/executorMain.c
浏览文件 @ ae2d77e9
......@@ -178,7 +178,9 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) {
publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC);
int64_t st = 0;
if (handle) {
*handle = pTaskInfo->dsHandle;
}
while(1) {
st = taosGetTimestampUs();
......
source/libs/executor/src/executorimpl.c
浏览文件 @ ae2d77e9
......@@ -18,7 +18,6 @@
#include "ttime.h"
#include "exception.h"
#include "../../../../contrib/cJson/cJSON.h"
#include "executorimpl.h"
#include "function.h"
#include "tcompare.h"
......@@ -295,7 +294,6 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO
SSDataBlock *res = calloc(1, sizeof(SSDataBlock));
res->info.numOfCols = numOfOutput;
res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData));
for (int32_t i = 0; i < numOfOutput; ++i) {
SColumnInfoData idata = {{0}};
......@@ -1816,7 +1814,7 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) {
return TSDB_CODE_SUCCESS;
}
static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
int32_t** rowCellInfoOffset) {
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
......@@ -1920,6 +1918,104 @@ static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI
return pFuncCtx;
}
static SQLFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) {
SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx));
if (pFuncCtx == NULL) {
return NULL;
}
*rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t));
if (*rowCellInfoOffset == 0) {
tfree(pFuncCtx);
return NULL;
}
for (int32_t i = 0; i < numOfOutput; ++i) {
SSqlExpr *pSqlExpr = &pExpr[i].base;
SQLFunctionCtx* pCtx = &pFuncCtx[i];
#if 0
SColIndex *pIndex = &pSqlExpr->colInfo;
if (TSDB_COL_REQ_NULL(pIndex->flag)) {
pCtx->requireNull = true;
pIndex->flag &= ~(TSDB_COL_NULL);
} else {
pCtx->requireNull = false;
}
#endif
// pCtx->inputBytes = pSqlExpr->colBytes;
// pCtx->inputType = pSqlExpr->colType;
pCtx->ptsOutputBuf = NULL;
pCtx->resDataInfo.bytes = pSqlExpr->resSchema.bytes;
pCtx->resDataInfo.type = pSqlExpr->resSchema.type;
// pCtx->order = pQueryAttr->order.order;
// pCtx->functionId = pSqlExpr->functionId;
// pCtx->stableQuery = pQueryAttr->stableQuery;
pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes;
pCtx->start.key = INT64_MIN;
pCtx->end.key = INT64_MIN;
pCtx->numOfParams = pSqlExpr->numOfParams;
for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
int16_t type = pSqlExpr->param[j].nType;
int16_t bytes = pSqlExpr->param[j].nLen;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
taosVariantCreateFromBinary(&pCtx->param[j], pSqlExpr->param[j].pz, bytes, type);
} else {
taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlExpr->param[j].i, bytes, type);
}
}
// set the order information for top/bottom query
int32_t functionId = pCtx->functionId;
if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
int32_t f = getExprFunctionId(&pExpr[0]);
assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY);
// pCtx->param[2].i = pQueryAttr->order.order;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[3].i = functionId;
pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT;
// pCtx->param[1].i = pQueryAttr->order.col.info.colId;
} else if (functionId == FUNCTION_INTERP) {
// pCtx->param[2].i = (int8_t)pQueryAttr->fillType;
// if (pQueryAttr->fillVal != NULL) {
// if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) {
// pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
// } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value
// if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
// taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType);
// }
// }
// }
} else if (functionId == FUNCTION_TS_COMP) {
// pCtx->param[0].i = pQueryAttr->vgId; //TODO this should be the parameter from client
pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT;
} else if (functionId == FUNCTION_TWA) {
// pCtx->param[1].i = pQueryAttr->window.skey;
pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT;
// pCtx->param[2].i = pQueryAttr->window.ekey;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
} else if (functionId == FUNCTION_ARITHM) {
// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
}
}
// for(int32_t i = 1; i < numOfOutput; ++i) {
// (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr[i - 1].base.interBytes);
// }
setCtxTagColumnInfo(pFuncCtx, numOfOutput);
return pFuncCtx;
}
static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) {
if (pCtx == NULL) {
return NULL;
......@@ -3468,8 +3564,6 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt
// }
// reverse order time range
SWAP(pTableScanInfo->window.skey, pTableScanInfo->window.ekey, TSKEY);
SET_REVERSE_SCAN_FLAG(pTableScanInfo);
// setTaskStatus(pTableScanInfo, QUERY_NOT_COMPLETED);
......@@ -4453,7 +4547,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// return true;
//}
void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) {
void appendDownstream(SOperatorInfo* p, SOperatorInfo* pUpstream) {
if (p->pDownstream == NULL) {
assert(p->numOfDownstream == 0);
}
......@@ -4546,28 +4640,6 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
pRuntimeEnv->cur.vgroupIndex = -1;
setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo);
switch(tbScanner) {
// case OP_TableBlockInfoScan: {
// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
// break;
// }
// case OP_TableSeqScan: {
// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
// break;
// }
// case OP_DataBlocksOptScan: {
// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
// break;
// }
// case OP_TableScan: {
// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr));
// break;
// }
default: { // do nothing
break;
}
}
if (sourceOptr != NULL) {
assert(pRuntimeEnv->proot == NULL);
pRuntimeEnv->proot = sourceOptr;
......@@ -4839,21 +4911,104 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) {
pOperator->status = OP_EXEC_DONE;
return pBlock;
#endif
}
int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) {
}
SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) {
static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
SOperatorInfo* pOperator = (SOperatorInfo*) param;
SExchangeInfo *pExchangeInfo = pOperator->info;
SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo;
*newgroup = false;
SResFetchReq *pMsg = calloc(1, sizeof(SResFetchReq));
if (NULL == pMsg) { // todo handle malloc error
}
SEpSet epSet;
int64_t sId = -1, queryId = 0, taskId = 1, vgId = 1;
pMsg->header.vgId = htonl(vgId);
pMsg->sId = htobe64(sId);
pMsg->taskId = htobe64(taskId);
pMsg->queryId = htobe64(queryId);
// send the fetch remote task result reques
SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", queryId, taskId, (int32_t)sizeof(SMsgSendInfo));
}
pMsgSendInfo->param = NULL;
pMsgSendInfo->msgInfo.pData = pMsg;
pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq);
pMsgSendInfo->msgType = TDMT_VND_FETCH;
pMsgSendInfo->fp = loadRemoteDataCallback;
int64_t transporterId = 0;
void* pTransporter = NULL;
int32_t code = asyncSendMsgToServer(pTransporter, &epSet, &transporterId, pMsgSendInfo);
printf("abc\n");
getchar();
// add it into the sink node
}
SOperatorInfo* createExchangeOperatorInfo(const SVgroupInfo* pVgroups, int32_t numOfSources, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
assert(numOfSources > 0);
SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo));
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
tfree(pInfo);
tfree(pOperator);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return NULL;
}
pInfo->numOfSources = numOfSources;
pOperator->name = "ExchangeOperator";
pOperator->operatorType = OP_Exchange;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = NULL;
pOperator->exec = doLoadRemoteData;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
}
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) {
assert(repeatTime > 0 && numOfOutput > 0);
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pTsdbReadHandle = pTsdbQueryHandle;
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
tfree(pInfo);
tfree(pOperator);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return NULL;
}
pInfo->pTsdbReadHandle = pTsdbReadHandle;
pInfo->times = repeatTime;
pInfo->reverseTimes = 0;
pInfo->order = order;
pInfo->current = 0;
pInfo->scanFlag = MAIN_SCAN;
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "TableScanOperator";
pOperator->operatorType = OP_TableScan;
pOperator->blockingOptr = false;
......@@ -4867,10 +5022,43 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in
return pOperator;
}
SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) {
SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) {
assert(repeatTime > 0);
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
tfree(pInfo);
tfree(pOperator);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return NULL;
}
pInfo->pTsdbReadHandle = pTsdbReadHandle;
pInfo->times = repeatTime;
pInfo->reverseTimes = reverseTime;
pInfo->order = order;
pInfo->current = 0;
pInfo->scanFlag = MAIN_SCAN;
pOperator->name = "DataBlocksOptimizedScanOperator";
pOperator->operatorType = OP_DataBlocksOptScan;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = NULL;
pOperator->exec = doTableScan;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
}
SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) {
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->pTsdbReadHandle = pTsdbReadHandle;
pInfo->times = 1;
pInfo->reverseTimes = 0;
pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
......@@ -4880,7 +5068,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "TableSeqScanOperator";
// pOperator->operatorType = OP_TableSeqScan;
pOperator->operatorType = OP_TableSeqScan;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
......@@ -4891,10 +5079,10 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn
return pOperator;
}
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) {
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) {
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->pTsdbReadHandle = pTsdbReadHandle;
pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
SColumnInfoData infoData = {{0}};
......@@ -4974,27 +5162,6 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf
}
SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) {
assert(repeatTime > 0);
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->times = repeatTime;
pInfo->reverseTimes = reverseTime;
pInfo->current = 0;
pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo));
pOptr->name = "DataBlocksOptimizedScanOperator";
// pOptr->operatorType = OP_DataBlocksOptScan;
pOptr->pRuntimeEnv = pRuntimeEnv;
pOptr->blockingOptr = false;
pOptr->info = pInfo;
pOptr->exec = doTableScan;
return pOptr;
}
SArray* getOrderCheckColumns(STaskAttr* pQuery) {
int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0: taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo);
......@@ -5099,7 +5266,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S
pInfo->bufCapacity = 4096;
pInfo->udfInfo = pUdfInfo;
pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor);
pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->orderColumnList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr);
pInfo->groupColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr);
......@@ -5148,7 +5315,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S
// pOperator->exec = doGlobalAggregate;
pOperator->cleanup = destroyGlobalAggOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -5297,7 +5464,7 @@ SOperatorInfo *createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
pOperator->cleanup = destroyOrderOperatorInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6229,33 +6396,33 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
tfree(pOperator);
}
SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo));
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery));
// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
int32_t numOfRows = 1;//(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery));
pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows);
pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT);
pInfo->seed = rand();
setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MAIN_SCAN);
// setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MAIN_SCAN);
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "TableAggregate";
// pOperator->operatorType = OP_Aggregate;
pOperator->operatorType = OP_Aggregate;
pOperator->blockingOptr = true;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->pRuntimeEnv = NULL;
pOperator->exec = doAggregate;
pOperator->cleanup = destroyAggOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6335,7 +6502,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOp
size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv);
pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup);
pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT);
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
......@@ -6350,7 +6517,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOp
pOperator->exec = doSTableAggregate;
pOperator->cleanup = destroyAggOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6363,7 +6530,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
SOptrBasicInfo* pBInfo = &pInfo->binfo;
pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity);
pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset);
pBInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset);
initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT);
setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN);
......@@ -6380,7 +6547,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
pOperator->exec = doProjectOperation;
pOperator->cleanup = destroyProjectOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6438,7 +6605,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanup = destroyConditionOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6456,7 +6623,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
pOperator->exec = doLimit;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6464,7 +6631,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);
pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);
pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT);
......@@ -6481,7 +6648,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe
pOperator->exec = doIntervalAgg;
pOperator->cleanup = destroyBasicOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6489,7 +6656,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);
pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);
pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT);
......@@ -6506,7 +6673,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S
pOperator->exec = doAllIntervalAgg;
pOperator->cleanup = destroyBasicOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6514,7 +6681,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper
SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo));
pInfo->colIndex = -1;
pInfo->reptScan = false;
pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT);
......@@ -6530,13 +6697,13 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper
pOperator->exec = doStateWindowAgg;
pOperator->cleanup = destroyStateWindowOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo));
pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT);
......@@ -6555,14 +6722,14 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
pOperator->exec = doSessionWindowAgg;
pOperator->cleanup = destroySWindowOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);
pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);
pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT);
......@@ -6579,14 +6746,14 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim
pOperator->exec = doSTableIntervalAgg;
pOperator->cleanup = destroyBasicOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);
pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);
pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity);
initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT);
......@@ -6603,7 +6770,7 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun
pOperator->exec = doAllSTableIntervalAgg;
pOperator->cleanup = destroyBasicOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6614,7 +6781,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
pInfo->colIndex = -1; // group by column index
pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
......@@ -6636,7 +6803,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
pOperator->exec = hashGroupbyAggregate;
pOperator->cleanup = destroyGroupbyOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6675,7 +6842,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf
pOperator->exec = doFill;
pOperator->cleanup = destroySFillOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -6723,7 +6890,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanup = destroySlimitOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -7021,7 +7188,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato
pOperator->pExpr = pExpr;
pOperator->cleanup = destroyDistinctOperatorInfo;
appendUpstream(pOperator, downstream);
appendDownstream(pOperator, downstream);
return pOperator;
}
......@@ -7188,31 +7355,49 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) {
SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) {
if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) {
if (pPhyNode->info.type == OP_TableScan) {
SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode;
size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets);
SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo);
pTaskInfo->pRoot = pOperatorInfo;
return createTableScanOperatorInfo(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo);
} else if (pPhyNode->info.type == OP_DataBlocksOptScan) {
SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode;
size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets);
return createDataBlocksOptScanInfo(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo);
} else {
assert(0);
}
}
}
int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) {
STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false};
cond.twindow.skey = INT64_MIN;
cond.twindow.ekey = INT64_MAX;
STsdbQueryCond cond = {.loadExternalRows = false};
uint64_t uid = 0;
SPhyNode* pPhyNode = pPlan->pNode;
if (pPhyNode->info.type == OP_TableScan || pPhyNode->info.type == OP_DataBlocksOptScan) {
STableScanPhyNode* pTableScanNode = (STableScanPhyNode*) pPhyNode;
uid = pTableScanNode->scan.uid;
cond.order = pTableScanNode->scan.order;
cond.numOfCols = taosArrayGetSize(pTableScanNode->scan.node.pTargets);
cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
cond.twindow = pTableScanNode->window;
// todo set the correct table column info
cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP;
cond.colList[0].bytes = sizeof(uint64_t);
cond.colList[0].colId = 1;
for(int32_t i = 0; i < cond.numOfCols; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pTableScanNode->scan.node.pTargets, i);
assert(pExprInfo->pExpr->nodeType == TEXPR_COL_NODE);
cond.colList[1].type = TSDB_DATA_TYPE_INT;
cond.colList[1].bytes = sizeof(int32_t);
cond.colList[1].colId = 2;
SSchema* pSchema = pExprInfo->pExpr->pSchema;
cond.colList[i].type = pSchema->type;
cond.colList[i].bytes = pSchema->bytes;
cond.colList[i].colId = pSchema->colId;
}
} else {
assert(0);
}
STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)};
SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo));
STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1};
STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid};
taosArrayPush(pa, &info);
taosArrayPush(group.pGroupList, &pa);
......@@ -7220,7 +7405,11 @@ int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* r
*pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId);
tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL);
doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle);
(*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle);
if ((*pTaskInfo)->pRoot == NULL) {
return terrno;
}
return TSDB_CODE_SUCCESS;
}
......
source/libs/function/src/tscalarfunction.c
浏览文件 @ ae2d77e9
......@@ -2,7 +2,6 @@
#include "tbinoperator.h"
#include "tunaryoperator.h"
static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) {
dst->type = src->type;
dst->bytes = src->bytes;
......
source/libs/parser/inc/astGenerator.h
浏览文件 @ ae2d77e9
......@@ -170,8 +170,6 @@ typedef struct SCreateDbInfo {
int8_t update;
int8_t cachelast;
SArray *keep;
// int8_t dbType;
// int16_t partitions;
} SCreateDbInfo;
typedef struct SCreateFuncInfo {
......
source/libs/parser/inc/astToMsg.h
浏览文件 @ ae2d77e9
......@@ -8,10 +8,10 @@
SCreateUserReq* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);
SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);
SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);
SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen);
SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf);
SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf);
SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf);
SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf);
SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf);
SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf);
......
source/libs/parser/inc/parserInt.h
浏览文件 @ ae2d77e9
......@@ -51,7 +51,7 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t
* @param msgBufLen
* @return
*/
int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen);
int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen);
/**
* validate the ddl ast, and convert the ast to the corresponding message format
......@@ -60,7 +60,7 @@ int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmt
* @param type
* @return
*/
SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen);
SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen);
/**
*
......@@ -70,7 +70,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c
* @param msgBufLen
* @return
*/
SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen);
SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen);
/**
* Evaluate the numeric and timestamp arithmetic expression in the WHERE clause.
......@@ -98,7 +98,7 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf);
* @param msgBufLen
* @return
*/
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseBasicCtx *pCtx, char* msg, int32_t msgBufLen);
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseContext *pCtx, char* msg, int32_t msgBufLen);
/**
* Destroy the meta data request structure.
......
source/libs/parser/inc/parserUtil.h
浏览文件 @ ae2d77e9
......@@ -81,7 +81,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param);
typedef int32_t (*_row_append_fn_t)(const void *value, int32_t len, void *param);
int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf);
int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf);
int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
#ifdef __cplusplus
}
......
source/libs/parser/src/astGenerator.c
浏览文件 @ ae2d77e9
......@@ -972,14 +972,15 @@ void tSetDbName(SToken *pCpxName, SToken *pDb) {
void tSetColumnInfo(SField *pField, SToken *pName, SField *pType) {
int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]);
// column name is too long, set the it to be invalid.
// The column name is too long, set it to be invalid.
if ((int32_t) pName->n >= maxLen) {
pName->n = -1;
pField->name[0] = 0;
} else {
strncpy(pField->name, pName->z, pName->n);
pField->name[pName->n] = 0;
}
// denote an invalid data type in the column definition.
pField->type = pType->type;
if(!isValidDataType(pField->type)){
pField->bytes = 0;
......
source/libs/parser/src/astToMsg.c
浏览文件 @ ae2d77e9
......@@ -85,8 +85,12 @@ SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, cha
return pMsg;
}
SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, int32_t msgLen) {
SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf) {
SShowReq* pShowMsg = calloc(1, sizeof(SShowReq));
if (pShowMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return pShowMsg;
}
pShowMsg->type = pShowInfo->showType;
if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
......@@ -105,7 +109,22 @@ SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf,
if (pShowInfo->showType == TSDB_MGMT_TABLE_STB || pShowInfo->showType == TSDB_MGMT_TABLE_VGROUP) {
SName n = {0};
if (pShowInfo->prefix.n > 0) {
if (pShowInfo->prefix.n >= TSDB_DB_FNAME_LEN) {
terrno = buildInvalidOperationMsg(pMsgBuf, "prefix name is too long");
tfree(pShowMsg);
return NULL;
}
tNameSetDbName(&n, pCtx->acctId, pShowInfo->prefix.z, pShowInfo->prefix.n);
} else if (pCtx->db == NULL || strlen(pCtx->db) == 0) {
terrno = buildInvalidOperationMsg(pMsgBuf, "database is not specified");
tfree(pShowMsg);
return NULL;
} else {
tNameSetDbName(&n, pCtx->acctId, pCtx->db, strlen(pCtx->db));
}
tNameGetFullDbName(&n, pShowMsg->db);
}
......@@ -210,7 +229,7 @@ int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbS
return TSDB_CODE_SUCCESS;
}
SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) {
SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf) {
SCreateDbReq* pCreateMsg = calloc(1, sizeof(SCreateDbReq));
if (setDbOptions(pCreateMsg, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) {
tfree(pCreateMsg);
......@@ -230,7 +249,7 @@ SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCt
return pCreateMsg;
}
SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) {
SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) {
SSchema* pSchema;
int32_t numOfTags = 0;
......@@ -240,6 +259,9 @@ SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len
}
SMCreateStbReq* pCreateStbMsg = (SMCreateStbReq*)calloc(1, sizeof(SMCreateStbReq) + (numOfCols + numOfTags) * sizeof(SSchema));
if (pCreateStbMsg == NULL) {
return NULL;
}
char* pMsg = NULL;
#if 0
......@@ -315,7 +337,7 @@ SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len
return pCreateStbMsg;
}
SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) {
SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) {
SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0);
SName name = {0};
......
source/libs/parser/src/astValidate.c
浏览文件 @ ae2d77e9
......@@ -917,6 +917,8 @@ int32_t validateLimitNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBu
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
}
return TSDB_CODE_SUCCESS;
}
int32_t validateOrderbyNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf* pMsgBuf) {
......@@ -3628,7 +3630,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf)
return TSDB_CODE_SUCCESS;
}
int32_t setTableVgroupList(SParseBasicCtx *pCtx, SName* name, SVgroupsInfo **pVgList) {
int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) {
SArray* vgroupList = NULL;
int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList);
if (code != TSDB_CODE_SUCCESS) {
......@@ -3655,7 +3657,7 @@ int32_t setTableVgroupList(SParseBasicCtx *pCtx, SName* name, SVgroupsInfo **pVg
return TSDB_CODE_SUCCESS;
}
int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) {
int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) {
assert(pCtx != NULL && pInfo != NULL);
int32_t code = 0;
......
source/libs/parser/src/dCDAstProcess.c
浏览文件 @ ae2d77e9
......@@ -18,7 +18,7 @@ static bool has(SArray* pFieldList, int32_t startIndex, const char* name) {
return false;
}
static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen,
static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** output, int32_t* outputLen,
SEpSet* pEpSet, void** pExtension, SMsgBuf* pMsgBuf) {
const char* msg1 = "invalid name";
const char* msg2 = "wildcard string should be less than %d characters";
......@@ -109,7 +109,11 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou
}
*pEpSet = pCtx->mgmtEpSet;
*output = buildShowMsg(pShowInfo, pCtx, pMsgBuf->buf, pMsgBuf->len);
*output = buildShowMsg(pShowInfo, pCtx, pMsgBuf);
if (*output == NULL) {
return terrno;
}
*outputLen = sizeof(SShowReq) /* + htons(pShowMsg->payloadLen)*/;
}
......@@ -312,9 +316,9 @@ int32_t doCheckForCreateTable(SCreateTableSql* pCreateTable, SMsgBuf* pMsgBuf) {
assert(pFieldList != NULL);
// if sql specifies db, use it, otherwise use default db
SToken* pzTableName = &(pCreateTable->name);
SToken* pNameToken = &(pCreateTable->name);
if (parserValidateNameToken(pzTableName) != TSDB_CODE_SUCCESS) {
if (parserValidateIdToken(pNameToken) != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
......@@ -396,7 +400,7 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) {
taosArrayDestroy(pTbBatch->req.pArray);
}
static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) {
static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseContext* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) {
const char* msg1 = "invalid table name";
const char* msg2 = "tags number not matched";
const char* msg3 = "tag value too long";
......@@ -634,7 +638,7 @@ static int32_t doBuildSingleTableBatchReq(SName* pTableName, SArray* pColumns, S
return TSDB_CODE_SUCCESS;
}
int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) {
int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseContext* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) {
SArray* pBufArray = NULL;
int32_t code = 0;
......@@ -703,7 +707,7 @@ SArray* doSerializeVgroupCreateTableInfo(SHashObj* pVgroupHashmap) {
return pBufArray;
}
SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) {
SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen) {
int32_t code = 0;
SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo));
......@@ -961,7 +965,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c
return NULL;
}
SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) {
SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen) {
SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
assert(pCreateTable->type == TSDB_SQL_CREATE_TABLE);
......@@ -973,6 +977,7 @@ SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBas
int32_t msgLen = 0;
int32_t code = doCheckAndBuildCreateTableReq(pCreateTable, pCtx, pMsgBuf, (char**) &pModifSqlStmt, &msgLen);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
tfree(pModifSqlStmt);
return NULL;
}
......
source/libs/parser/src/insertParser.c
浏览文件 @ ae2d77e9
......@@ -84,11 +84,11 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD
char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false);
if (NULL != p) { // db.table
int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->ctx.acctId);
int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->acctId);
strncpy(fullDbName + n, pStname->z, p - pStname->z);
strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1);
} else {
snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->ctx.acctId, pCxt->pComCxt->ctx.db);
snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->acctId, pCxt->pComCxt->db);
strncpy(tableName, pStname->z, pStname->n);
}
......@@ -97,11 +97,11 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD
static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
SName name = {0};
createSName(&name, pTname, &pCxt->pComCxt->ctx, &pCxt->msg);
createSName(&name, pTname, pCxt->pComCxt, &pCxt->msg);
char tableName[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&name, tableName);
SParseBasicCtx* pBasicCtx = &pCxt->pComCxt->ctx;
SParseContext* pBasicCtx = pCxt->pComCxt;
CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta));
SVgroupInfo vg;
CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg));
......
source/libs/parser/src/parser.c
浏览文件 @ ae2d77e9
......@@ -45,14 +45,14 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
if (!isDqlSqlStatement(&info)) {
if (info.type == TSDB_SQL_CREATE_TABLE) {
SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen);
SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen);
if (pModifStmtInfo == NULL) {
return terrno;
}
*pQuery = (SQueryNode*)pModifStmtInfo;
} else {
SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen);
SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen);
if (pDcl == NULL) {
return terrno;
}
......@@ -67,7 +67,7 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
return terrno;
}
int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen);
int32_t code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen);
if (code == TSDB_CODE_SUCCESS) {
*pQuery = (SQueryNode*)pQueryInfo;
} else {
......@@ -80,11 +80,11 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
return TSDB_CODE_SUCCESS;
}
int32_t qParseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
int32_t qParseQuerySql(SParseContext* pCxt, SQueryNode** pQueryNode) {
if (isInsertSql(pCxt->pSql, pCxt->sqlLen)) {
return parseInsertSql(pCxt, (SVnodeModifOpStmtInfo**)pQuery);
return parseInsertSql(pCxt, (SVnodeModifOpStmtInfo**)pQueryNode);
} else {
return parseQuerySql(pCxt, pQuery);
return parseQuerySql(pCxt, pQueryNode);
}
}
......@@ -92,7 +92,7 @@ int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) {
return 0;
}
static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf);
static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pCtx, SMsgBuf* pMsgBuf);
static int32_t tnameComparFn(const void* p1, const void* p2) {
SName* pn1 = (SName*)p1;
......@@ -116,7 +116,7 @@ static int32_t tnameComparFn(const void* p1, const void* p2) {
}
}
static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) {
static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pCtx, SMsgBuf* pMsgBuf) {
int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list);
for (int32_t j = 0; j < numOfSub; ++j) {
......@@ -139,7 +139,7 @@ static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameLis
return TSDB_CODE_SUCCESS;
}
int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pParseCtx, SMsgBuf* pMsgBuf) {
int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pParseCtx, SMsgBuf* pMsgBuf) {
const char* msg1 = "invalid table name";
int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list);
......@@ -173,7 +173,7 @@ static void freePtrElem(void* p) {
tfree(*(char**)p);
}
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseBasicCtx *pCtx, char* msg, int32_t msgBufLen) {
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseContext *pCtx, char* msg, int32_t msgBufLen) {
int32_t code = TSDB_CODE_SUCCESS;
SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen};
......
source/libs/parser/src/parserUtil.c
浏览文件 @ ae2d77e9
......@@ -124,12 +124,13 @@ int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) {
}
int32_t parserValidateNameToken(SToken* pToken) {
if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) {
if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID || pToken->n == 0) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
// it is a token quoted with escape char '`'
if (pToken->z[0] == TS_ESCAPE_CHAR && pToken->z[pToken->n - 1] == TS_ESCAPE_CHAR) {
pToken->n = strdequote(pToken->z);
return TSDB_CODE_SUCCESS;
}
......@@ -1943,19 +1944,32 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param) {
return TSDB_CODE_SUCCESS;
}
int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) {
int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) {
const char* msg1 = "name too long";
const char* msg2 = "invalid database name";
int32_t code = TSDB_CODE_SUCCESS;
char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, false);
char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, true);
if (p != NULL) { // db has been specified in sql string so we ignore current db path
tNameSetAcctId(pName, pParseCtx->acctId);
assert(*p == TS_PATH_DELIMITER[0]);
char name[TSDB_TABLE_FNAME_LEN] = {0};
strncpy(name, pTableName->z, pTableName->n);
int32_t dbLen = p - pTableName->z;
char name[TSDB_DB_FNAME_LEN] = {0};
strncpy(name, pTableName->z, dbLen);
dbLen = strdequote(name);
code = tNameSetDbName(pName, pParseCtx->acctId, name, dbLen);
if (code != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
int32_t tbLen = pTableName->n - dbLen - 1;
char tbname[TSDB_TABLE_FNAME_LEN] = {0};
strncpy(tbname, p + 1, tbLen);
/*tbLen = */strdequote(tbname);
code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE);
code = tNameFromString(pName, tbname, T_NAME_TABLE);
if (code != 0) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
......@@ -1964,10 +1978,17 @@ int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx,
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db));
assert(pTableName->n < TSDB_TABLE_FNAME_LEN);
char name[TSDB_TABLE_FNAME_LEN] = {0};
strncpy(name, pTableName->z, pTableName->n);
strdequote(name);
code = tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db));
if (code != TSDB_CODE_SUCCESS) {
code = buildInvalidOperationMsg(pMsgBuf, msg2);
return code;
}
code = tNameFromString(pName, name, T_NAME_TABLE);
if (code != 0) {
......
source/libs/parser/src/queryInfoUtil.c
浏览文件 @ ae2d77e9
......@@ -160,11 +160,14 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t
taosArrayInsert(pExprList, index, &pExprInfo);
}
#if 0
if (pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE) {
printf("add function: %s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList));
} else {
printf("add operator: %s, level:%d, total:%ld\n", pExprInfo->base.resSchema.name, level, taosArrayGetSize(pExprList));
}
#endif
}
void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) {
......
source/libs/parser/test/insertParserTest.cpp
浏览文件 @ ae2d77e9
......@@ -43,8 +43,8 @@ protected:
void bind(const char* sql) {
reset();
cxt_.ctx.acctId = atoi(acctId_.c_str());
cxt_.ctx.db = (char*) db_.c_str();
cxt_.acctId = atoi(acctId_.c_str());
cxt_.db = (char*) db_.c_str();
strcpy(sqlBuf_, sql);
cxt_.sqlLen = strlen(sql);
sqlBuf_[cxt_.sqlLen] = '\0';
......
source/libs/parser/test/parserTests.cpp
浏览文件 @ ae2d77e9
......@@ -77,7 +77,7 @@ void sqlCheck(const char* sql, bool valid) {
buf.len = 128;
buf.buf = msg;
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0);
......@@ -122,7 +122,7 @@ TEST(testCase, validateAST_test) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -184,7 +184,7 @@ TEST(testCase, function_Test) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -234,7 +234,7 @@ TEST(testCase, function_Test2) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -284,7 +284,7 @@ TEST(testCase, function_Test3) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -333,7 +333,7 @@ TEST(testCase, function_Test4) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -385,7 +385,7 @@ TEST(testCase, function_Test5) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -474,7 +474,7 @@ TEST(testCase, function_Test6) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -556,7 +556,7 @@ TEST(testCase, function_Test6) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -622,7 +622,7 @@ TEST(testCase, function_Test6) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -705,7 +705,7 @@ TEST(testCase, function_Test6) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......@@ -756,7 +756,7 @@ TEST(testCase, show_user_Test) {
SSqlInfo info1 = doGenerateAST(sql1);
ASSERT_EQ(info1.valid, true);
SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL};
SParseContext ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL};
SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len);
ASSERT_NE(output, nullptr);
......@@ -776,7 +776,7 @@ TEST(testCase, create_user_Test) {
ASSERT_EQ(info1.valid, true);
ASSERT_EQ(isDclSqlStatement(&info1), true);
SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc"};
SParseContext ct= {.requestId = 1, .acctId = 1, .db = "abc"};
SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len);
ASSERT_NE(output, nullptr);
......
source/libs/parser/test/plannerTest.cpp
浏览文件 @ ae2d77e9
......@@ -81,7 +81,7 @@ void generateLogicplan(const char* sql) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
ASSERT_EQ(ret, 0);
ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
......@@ -122,7 +122,7 @@ TEST(testCase, planner_test) {
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
ASSERT_EQ(ret, 0);
......
source/libs/parser/test/tokenizerTest.cpp
浏览文件 @ ae2d77e9
......@@ -711,7 +711,7 @@ TEST(testCase, extractMeta_test) {
char msg[128] = {0};
SCatalogReq req = {0};
SParseBasicCtx ctx = {0};
SParseContext ctx = {0};
ctx.db = "db1";
ctx.acctId = 1;
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128);
......
source/libs/planner/src/logicPlan.c
浏览文件 @ ae2d77e9
......@@ -38,10 +38,10 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) {
}
static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) {
SVnodeModifOpStmtInfo* pInsert = (SVnodeModifOpStmtInfo*)pNode;
SVnodeModifOpStmtInfo* pModifStmtInfo = (SVnodeModifOpStmtInfo*)pNode;
*pQueryPlan = calloc(1, sizeof(SQueryPlanNode));
SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES);
SArray* blocks = taosArrayInit(taosArrayGetSize(pModifStmtInfo->pDataBlocks), POINTER_BYTES);
SDataPayloadInfo* pPayload = calloc(1, sizeof(SDataPayloadInfo));
if (NULL == *pQueryPlan || NULL == blocks || NULL == pPayload) {
......@@ -49,7 +49,7 @@ static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode*
}
(*pQueryPlan)->info.type = QNODE_MODIFY;
taosArrayAddAll(blocks, pInsert->pDataBlocks);
taosArrayAddAll(blocks, pModifStmtInfo->pDataBlocks);
if (pNode->type == TSDB_SQL_INSERT) {
pPayload->msgType = TDMT_VND_SUBMIT;
......
source/libs/planner/src/physicalPlan.c
浏览文件 @ ae2d77e9
......@@ -156,9 +156,14 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si
}
static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) {
SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size);
node->uid = pTable->pMeta->pTableMeta->uid;
node->tableType = pTable->pMeta->pTableMeta->tableType;
SScanPhyNode* node = (SScanPhyNode*) initPhyNode(pPlanNode, type, size);
STableMeta *pTableMeta = pTable->pMeta->pTableMeta;
node->uid = pTableMeta->uid;
node->count = 1;
node->order = TSDB_ORDER_ASC;
node->tableType = pTableMeta->tableType;
return (SPhyNode*)node;
}
......@@ -176,10 +181,10 @@ static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
return MAIN_SCAN;
}
static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) {
STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode));
node->scanFlag = getScanFlag(pPlanNode, pTable);
node->window = pTable->window;
static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pQueryTableInfo, int32_t op) {
STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pQueryTableInfo, op, sizeof(STableScanPhyNode));
node->scanFlag = getScanFlag(pPlanNode, pQueryTableInfo);
node->window = pQueryTableInfo->window;
// todo tag cond
return (SPhyNode*)node;
}
......@@ -257,8 +262,8 @@ static void vgroupMsgToEpSet(const SVgroupMsg* vg, SQueryNodeAddr* execNode) {
}
static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList;
for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) {
SVgroupsInfo* pVgroupList = pTable->pMeta->vgroupList;
for (int32_t i = 0; i < pVgroupList->numOfVgroups; ++i) {
STORE_CURRENT_SUBPLAN(pCxt);
SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN);
subplan->msgType = TDMT_VND_QUERY;
......@@ -290,13 +295,31 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl
static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo;
if (needMultiNodeScan(pTable)) {
return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable));
}
return createSingleTableScanNode(pPlanNode, pTable, pCxt->pCurrentSubplan);
}
static SPhyNode* createSingleTableAgg(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
SAggPhyNode* node = (SAggPhyNode*)initPhyNode(pPlanNode, OP_Aggregate, sizeof(SAggPhyNode));
SGroupbyExpr* pGroupBy = (SGroupbyExpr*)pPlanNode->pExtInfo;
node->aggAlgo = AGG_ALGO_PLAIN;
node->aggSplit = AGG_SPLIT_FINAL;
if (NULL != pGroupBy) {
node->aggAlgo = AGG_ALGO_HASHED;
node->pGroupByList = validPointer(taosArrayDup(pGroupBy->columnInfo));
}
return (SPhyNode*)node;
}
static SPhyNode* createAggNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
// if (needMultiNodeAgg(pPlanNode)) {
// }
return createSingleTableAgg(pCxt, pPlanNode);
}
static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
SPhyNode* node = NULL;
switch (pPlanNode->info.type) {
......@@ -306,6 +329,10 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
case QNODE_TABLESCAN:
node = createTableScanNode(pCxt, pPlanNode);
break;
case QNODE_AGGREGATE:
case QNODE_GROUPBY:
node = createAggNode(pCxt, pPlanNode);
break;
case QNODE_MODIFY:
// Insert is not an operator in a physical plan.
break;
......
source/libs/planner/src/physicalPlanJson.c
浏览文件 @ ae2d77e9
......@@ -20,6 +20,19 @@
typedef bool (*FToJson)(const void* obj, cJSON* json);
typedef bool (*FFromJson)(const cJSON* json, void* obj);
static char* getString(const cJSON* json, const char* name) {
char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name));
return strdup(p);
}
static void copyString(const cJSON* json, const char* name, char* dst) {
strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name)));
}
static int64_t getNumber(const cJSON* json, const char* name) {
return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name));
}
static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) {
if (NULL == obj) {
return true;
......@@ -62,6 +75,39 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f
return func(jObj, *obj);
}
static const char* jkPnodeType = "Type";
static int32_t getPnodeTypeSize(cJSON* json) {
switch (getNumber(json, jkPnodeType)) {
case OP_TableScan:
case OP_DataBlocksOptScan:
case OP_TableSeqScan:
return sizeof(STableScanPhyNode);
case OP_TagScan:
return sizeof(STagScanPhyNode);
case OP_SystemTableScan:
return sizeof(SSystemTableScanPhyNode);
case OP_Aggregate:
return sizeof(SAggPhyNode);
case OP_Exchange:
return sizeof(SExchangePhyNode);
default:
break;
};
return -1;
}
static bool fromPnode(const cJSON* json, const char* name, FFromJson func, void** obj) {
cJSON* jObj = cJSON_GetObjectItem(json, name);
if (NULL == jObj) {
return true;
}
*obj = calloc(1, getPnodeTypeSize(jObj));
if (NULL == *obj) {
return false;
}
return func(jObj, *obj);
}
static bool addTarray(cJSON* json, const char* name, FToJson func, const SArray* array, bool isPoint) {
size_t size = (NULL == array) ? 0 : taosArrayGetSize(array);
if (size > 0) {
......@@ -136,7 +182,7 @@ static const cJSON* getArray(const cJSON* json, const char* name, int32_t* size)
static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t itemSize, int32_t size) {
for (int32_t i = 0; i < size; ++i) {
if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize)) {
if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize * i)) {
return false;
}
}
......@@ -154,28 +200,9 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson
return fromItem(jArray, func, *array, itemSize, *size);
}
static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) {
static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) {
const cJSON* jArray = getArray(json, name, size);
if (*array == NULL) {
*array = calloc(*size, itemSize);
}
return fromItem(jArray, func, *array, itemSize, *size);
}
static char* getString(const cJSON* json, const char* name) {
char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name));
char* res = calloc(1, strlen(p) + 1);
strcpy(res, p);
return res;
}
static void copyString(const cJSON* json, const char* name, char* dst) {
strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name)));
}
static int64_t getNumber(const cJSON* json, const char* name) {
return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name));
return fromItem(jArray, func, array, itemSize, *size);
}
static const char* jkSchemaType = "Type";
......@@ -223,7 +250,7 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) {
schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize);
schema->precision = getNumber(json, jkDataBlockSchemaPrecision);
return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**) &(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols);
return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols);
}
static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr";
......@@ -530,23 +557,100 @@ static bool timeWindowFromJson(const cJSON* json, void* obj) {
static const char* jkScanNodeTableId = "TableId";
static const char* jkScanNodeTableType = "TableType";
static const char* jkScanNodeTableOrder = "Order";
static const char* jkScanNodeTableCount = "Count";
static const char* jkScanNodeTableRevCount = "Reverse";
static bool scanNodeToJson(const void* obj, cJSON* json) {
const SScanPhyNode* scan = (const SScanPhyNode*)obj;
bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid);
const SScanPhyNode* pNode = (const SScanPhyNode*)obj;
bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, pNode->uid);
if (res) {
res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType);
res = cJSON_AddNumberToObject(json, jkScanNodeTableType, pNode->tableType);
}
if (res) {
res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, pNode->order);
}
if (res) {
res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, pNode->count);
}
if (res) {
res = cJSON_AddNumberToObject(json, jkScanNodeTableRevCount, pNode->reverse);
}
return res;
}
static bool scanNodeFromJson(const cJSON* json, void* obj) {
SScanPhyNode* scan = (SScanPhyNode*)obj;
scan->uid = getNumber(json, jkScanNodeTableId);
scan->tableType = getNumber(json, jkScanNodeTableType);
SScanPhyNode* pNode = (SScanPhyNode*)obj;
pNode->uid = getNumber(json, jkScanNodeTableId);
pNode->tableType = getNumber(json, jkScanNodeTableType);
pNode->count = getNumber(json, jkScanNodeTableCount);
pNode->order = getNumber(json, jkScanNodeTableOrder);
pNode->reverse = getNumber(json, jkScanNodeTableRevCount);
return true;
}
static const char* jkColIndexColId = "ColId";
static const char* jkColIndexColIndex = "ColIndex";
static const char* jkColIndexFlag = "Flag";
static const char* jkColIndexName = "Name";
static bool colIndexToJson(const void* obj, cJSON* json) {
const SColIndex* col = (const SColIndex*)obj;
bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId);
if (res) {
res = cJSON_AddNumberToObject(json, jkColIndexColIndex, col->colIndex);
}
if (res) {
res = cJSON_AddNumberToObject(json, jkColIndexFlag, col->flag);
}
if (res) {
res = cJSON_AddStringToObject(json, jkColIndexName, col->name);
}
return res;
}
static bool colIndexFromJson(const cJSON* json, void* obj) {
SColIndex* col = (SColIndex*)obj;
col->colId = getNumber(json, jkColIndexColId);
col->colIndex = getNumber(json, jkColIndexColIndex);
col->flag = getNumber(json, jkColIndexFlag);
copyString(json, jkColIndexName, col->name);
return true;
}
static const char* jkAggNodeAggAlgo = "AggAlgo";
static const char* jkAggNodeAggSplit = "AggSplit";
static const char* jkAggNodeExprs = "Exprs";
static const char* jkAggNodeGroupByList = "GroupByList";
static bool aggNodeToJson(const void* obj, cJSON* json) {
const SAggPhyNode* agg = (const SAggPhyNode*)obj;
bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo);
if (res) {
res = cJSON_AddNumberToObject(json, jkAggNodeAggSplit, agg->aggSplit);
}
if (res) {
res = addArray(json, jkAggNodeExprs, exprInfoToJson, agg->pExprs);
}
if (res) {
res = addArray(json, jkAggNodeGroupByList, colIndexToJson, agg->pGroupByList);
}
return res;
}
static bool aggNodeFromJson(const cJSON* json, void* obj) {
SAggPhyNode* agg = (SAggPhyNode*)obj;
agg->aggAlgo = getNumber(json, jkAggNodeAggAlgo);
agg->aggSplit = getNumber(json, jkAggNodeAggSplit);
bool res = fromArray(json, jkAggNodeExprs, exprInfoFromJson, &agg->pExprs, sizeof(SExprInfo));
if (res) {
res = fromArray(json, jkAggNodeGroupByList, colIndexFromJson, &agg->pGroupByList, sizeof(SExprInfo));
}
return res;
}
static const char* jkTableScanNodeFlag = "Flag";
static const char* jkTableScanNodeWindow = "Window";
static const char* jkTableScanNodeTagsConditions = "TagsConditions";
......@@ -655,10 +759,10 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) {
case OP_SystemTableScan:
return scanNodeToJson(obj, json);
case OP_Aggregate:
break; // todo
return aggNodeToJson(obj, json);
case OP_Project:
return true;
case OP_Groupby:
// case OP_Groupby:
case OP_Limit:
case OP_SLimit:
case OP_TimeWindow:
......@@ -696,7 +800,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) {
break; // todo
case OP_Project:
return true;
case OP_Groupby:
// case OP_Groupby:
case OP_Limit:
case OP_SLimit:
case OP_TimeWindow:
......@@ -723,12 +827,15 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) {
static const char* jkPnodeName = "Name";
static const char* jkPnodeTargets = "Targets";
static const char* jkPnodeConditions = "Conditions";
static const char* jkPnodeSchema = "InputSchema";
static const char* jkPnodeSchema = "TargetSchema";
static const char* jkPnodeChildren = "Children";
// The 'pParent' field do not need to be serialized.
static bool phyNodeToJson(const void* obj, cJSON* jNode) {
const SPhyNode* phyNode = (const SPhyNode*)obj;
bool res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name);
bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type);
if (res) {
res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name);
}
if (res) {
res = addArray(jNode, jkPnodeTargets, exprInfoToJson, phyNode->pTargets);
}
......@@ -748,9 +855,11 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) {
}
static bool phyNodeFromJson(const cJSON* json, void* obj) {
SPhyNode* node = (SPhyNode*)obj;
node->info.name = getString(json, jkPnodeName);
node->info.type = opNameToOpType(node->info.name);
SPhyNode* node = (SPhyNode*) obj;
node->info.type = getNumber(json, jkPnodeType);
node->info.name = opTypeToOpName(node->info.type);
bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo));
if (res) {
res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo));
......@@ -894,10 +1003,13 @@ static SSubplan* subplanFromJson(const cJSON* json) {
if (NULL == subplan) {
return NULL;
}
bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true);
if (res) {
res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, sizeof(SPhyNode), false);
res = fromPnode(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode);
}
if (res) {
res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false);
}
......@@ -925,8 +1037,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) {
}
*str = cJSON_Print(json);
printf("%s\n", *str);
// printf("====Physical plan:====\n");
// printf("%s\n", *str);
*len = strlen(*str) + 1;
return TSDB_CODE_SUCCESS;
}
......@@ -945,14 +1057,18 @@ cJSON* qDagToJson(const SQueryDag* pDag) {
if(pRoot == NULL) {
return NULL;
}
cJSON_AddNumberToObject(pRoot, "numOfSubplans", pDag->numOfSubplans);
cJSON_AddNumberToObject(pRoot, "queryId", pDag->queryId);
cJSON_AddNumberToObject(pRoot, "Number", pDag->numOfSubplans);
cJSON_AddNumberToObject(pRoot, "QueryId", pDag->queryId);
cJSON *pLevels = cJSON_CreateArray();
if(pLevels == NULL) {
cJSON_Delete(pRoot);
return NULL;
}
cJSON_AddItemToObject(pRoot, "pSubplans", pLevels);
cJSON_AddItemToObject(pRoot, "Subplans", pLevels);
size_t level = taosArrayGetSize(pDag->pSubplans);
for(size_t i = 0; i < level; i++) {
const SArray* pSubplans = (const SArray*)taosArrayGetP(pDag->pSubplans, i);
......@@ -962,6 +1078,7 @@ cJSON* qDagToJson(const SQueryDag* pDag) {
cJSON_Delete(pRoot);
return NULL;
}
cJSON_AddItemToArray(pLevels, plansOneLevel);
for(size_t j = 0; j < num; j++) {
cJSON* pSubplan = subplanToJson((const SSubplan*)taosArrayGetP(pSubplans, j));
......@@ -969,6 +1086,7 @@ cJSON* qDagToJson(const SQueryDag* pDag) {
cJSON_Delete(pRoot);
return NULL;
}
cJSON_AddItemToArray(plansOneLevel, pSubplan);
}
}
......
source/libs/planner/test/phyPlanTests.cpp
浏览文件 @ ae2d77e9
......@@ -30,6 +30,21 @@ void* myCalloc(size_t nmemb, size_t size) {
class PhyPlanTest : public Test {
protected:
void pushAgg(int32_t aggOp) {
unique_ptr<SQueryPlanNode> agg((SQueryPlanNode*)myCalloc(1, sizeof(SQueryPlanNode)));
agg->info.type = aggOp;
agg->pExpr = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
unique_ptr<SExprInfo> expr((SExprInfo*)myCalloc(1, sizeof(SExprInfo)));
expr->base.resSchema.type = TSDB_DATA_TYPE_INT;
expr->base.resSchema.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
expr->pExpr = (tExprNode*)myCalloc(1, sizeof(tExprNode));
expr->pExpr->nodeType = TEXPR_FUNCTION_NODE;
strcpy(expr->pExpr->_function.functionName, "Count");
SExprInfo* item = expr.release();
taosArrayPush(agg->pExpr, &item);
pushNode(agg.release());
}
void pushScan(const string& db, const string& table, int32_t scanOp) {
shared_ptr<MockTableMeta> meta = mockCatalogService->getTableMeta(db, table);
EXPECT_TRUE(meta);
......@@ -95,10 +110,11 @@ protected:
private:
void pushNode(SQueryPlanNode* node) {
if (logicPlan_) {
// todo
} else {
logicPlan_.reset(node);
node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
SQueryPlanNode* child = logicPlan_.release();
taosArrayPush(node->pChildren, &child);
}
logicPlan_.reset(node);
}
void copySchemaMeta(STableMeta** dst, const STableMeta* src) {
......@@ -135,9 +151,9 @@ private:
_sql = sql;
memset(_msg, 0, _msgMaxLen);
pCxt->ctx.acctId = 1;
pCxt->ctx.db = _db.c_str();
pCxt->ctx.requestId = 1;
pCxt->acctId = 1;
pCxt->db = _db.c_str();
pCxt->requestId = 1;
pCxt->pSql = _sql.c_str();
pCxt->sqlLen = _sql.length();
pCxt->pMsg = _msg;
......@@ -174,6 +190,16 @@ TEST_F(PhyPlanTest, superTableScanTest) {
// todo check
}
// select count(*) from table
TEST_F(PhyPlanTest, simpleAggTest) {
pushScan("test", "t1", QNODE_TABLESCAN);
pushAgg(QNODE_AGGREGATE);
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
explain();
SQueryDag* dag = result();
// todo check
}
// insert into t values(...)
TEST_F(PhyPlanTest, insertTest) {
ASSERT_EQ(run("test", "insert into t1 values (now, 1, \"beijing\")"), TSDB_CODE_SUCCESS);
......
source/libs/qworker/inc/qworkerInt.h
浏览文件 @ ae2d77e9
......@@ -23,7 +23,7 @@ extern "C" {
#include "tlockfree.h"
#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000
#define QWORKER_DEFAULT_RES_CACHE_NUMBER 10000
#define QWORKER_DEFAULT_TASK_NUMBER 10000
#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000
enum {
......@@ -57,7 +57,6 @@ enum {
QW_ADD_ACQUIRE,
};
typedef struct SQWTaskStatus {
SRWLatch lock;
int32_t code;
......@@ -67,12 +66,15 @@ typedef struct SQWTaskStatus {
bool drop;
} SQWTaskStatus;
typedef struct SQWorkerTaskHandlesCache {
typedef struct SQWTaskCtx {
SRWLatch lock;
int8_t sinkScheduled;
int8_t queryScheduled;
bool needRsp;
qTaskInfo_t taskHandle;
DataSinkHandle sinkHandle;
} SQWorkerTaskHandlesCache;
} SQWTaskCtx;
typedef struct SQWSchStatus {
int32_t lastAccessTs; // timestamp in second
......@@ -83,10 +85,14 @@ typedef struct SQWSchStatus {
// Qnode/Vnode level task management
typedef struct SQWorkerMgmt {
SQWorkerCfg cfg;
int8_t nodeType;
int32_t nodeId;
SRWLatch schLock;
SRWLatch resLock;
SRWLatch ctxLock;
SHashObj *schHash; //key: schedulerId, value: SQWSchStatus
SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache
SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx
void *nodeObj;
putReqToQueryQFp putToQueueFp;
} SQWorkerMgmt;
#define QW_GOT_RES_DATA(data) (true)
......@@ -94,41 +100,69 @@ typedef struct SQWorkerMgmt {
#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code))
#define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code))
#define QW_TASK_READY_RESP(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED)
#define QW_SET_QTID(id, qid, tid) do { *(uint64_t *)(id) = (qid); *(uint64_t *)((char *)(id) + sizeof(qid)) = (tid); } while (0)
#define QW_GET_QTID(id, qid, tid) do { (qid) = *(uint64_t *)(id); (tid) = *(uint64_t *)((char *)(id) + sizeof(qid)); } while (0)
#define QW_TASK_READY(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED)
#define QW_SET_QTID(id, qId, tId) do { *(uint64_t *)(id) = (qId); *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); } while (0)
#define QW_GET_QTID(id, qId, tId) do { (qId) = *(uint64_t *)(id); (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); } while (0)
#define QW_IDS() sId, qId, tId
#define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0)
#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__)
#define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__)
#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000
#define QW_LOCK(type, _lock) do { \
if (QW_READ == (type)) { \
if ((*(_lock)) < 0) assert(0); \
assert(atomic_load_32((_lock)) >= 0); \
qDebug("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRLockLatch(_lock); \
qDebug("QW RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
qDebug("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) > 0); \
} else { \
if ((*(_lock)) < 0) assert(0); \
assert(atomic_load_32((_lock)) >= 0); \
qDebug("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWLockLatch(_lock); \
qDebug("QW WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
qDebug("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
} \
} while (0)
#define QW_UNLOCK(type, _lock) do { \
if (QW_READ == (type)) { \
if ((*(_lock)) <= 0) assert(0); \
assert(atomic_load_32((_lock)) > 0); \
qDebug("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRUnLockLatch(_lock); \
qDebug("QW RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
qDebug("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} else { \
if ((*(_lock)) <= 0) assert(0); \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
qDebug("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWUnLockLatch(_lock); \
qDebug("QW WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
qDebug("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} \
} while (0)
static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt);
int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch);
int32_t qwAcquireAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch);
int32_t qwAcquireTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task);
#ifdef __cplusplus
......
source/libs/qworker/src/qworker.c
浏览文件 @ ae2d77e9
......@@ -8,7 +8,7 @@
#include "tname.h"
#include "dataSinkMgt.h"
int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) {
int32_t qwValidateStatus(SQWorkerMgmt *mgmt, int8_t oriStatus, int8_t newStatus, uint64_t sId, uint64_t qId, uint64_t tId) {
int32_t code = 0;
if (oriStatus == newStatus) {
......@@ -62,7 +62,7 @@ int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) {
break;
default:
qError("invalid task status:%d", oriStatus);
QW_TASK_ELOG("invalid task status:%d", oriStatus);
return TSDB_CODE_QRY_APP_ERROR;
}
......@@ -70,22 +70,27 @@ int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) {
_return:
qError("invalid task status, from %d to %d", oriStatus, newStatus);
QW_ERR_RET(code);
QW_TASK_ELOG("invalid task status update from %d to %d", oriStatus, newStatus);
QW_RET(code);
}
int32_t qwUpdateTaskInfo(SQWTaskStatus *task, int8_t type, void *data) {
int32_t qwUpdateTaskInfo(SQWorkerMgmt *mgmt, SQWTaskStatus *task, int8_t type, void *data, uint64_t sId, uint64_t qId, uint64_t tId) {
int32_t code = 0;
int8_t origStatus = 0;
switch (type) {
case QW_TASK_INFO_STATUS: {
int8_t newStatus = *(int8_t *)data;
QW_ERR_RET(qwValidateStatus(task->status, newStatus));
QW_ERR_RET(qwValidateStatus(mgmt, task->status, newStatus, QW_IDS()));
origStatus = task->status;
task->status = newStatus;
QW_TASK_DLOG("task status updated from %d to %d", origStatus, newStatus);
break;
}
default:
qError("uknown task info type:%d", type);
QW_TASK_ELOG("unknown task info, type:%d", type);
return TSDB_CODE_QRY_APP_ERROR;
}
......@@ -96,27 +101,27 @@ int32_t qwAddTaskHandlesToCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId,
char id[sizeof(qId) + sizeof(tId)] = {0};
QW_SET_QTID(id, qId, tId);
SQWorkerTaskHandlesCache resCache = {0};
SQWTaskCtx resCache = {0};
resCache.taskHandle = taskHandle;
resCache.sinkHandle = sinkHandle;
QW_LOCK(QW_WRITE, &mgmt->resLock);
if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerTaskHandlesCache))) {
QW_UNLOCK(QW_WRITE, &mgmt->resLock);
qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", qId, tId);
QW_LOCK(QW_WRITE, &mgmt->ctxLock);
if (0 != taosHashPut(mgmt->ctxHash, id, sizeof(id), &resCache, sizeof(SQWTaskCtx))) {
QW_UNLOCK(QW_WRITE, &mgmt->ctxLock);
QW_TASK_ELOG("taosHashPut task ctx to ctxHash failed, taskHandle:%p, sinkHandle:%p", taskHandle, sinkHandle);
return TSDB_CODE_QRY_APP_ERROR;
}
QW_UNLOCK(QW_WRITE, &mgmt->resLock);
QW_UNLOCK(QW_WRITE, &mgmt->ctxLock);
return TSDB_CODE_SUCCESS;
}
static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) {
int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) {
SQWSchStatus newSch = {0};
newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (NULL == newSch.tasksHash) {
qError("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum);
QW_SCH_DLOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
......@@ -126,14 +131,18 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId,
if (0 != code) {
if (!HASH_NODE_EXIST(code)) {
QW_UNLOCK(QW_WRITE, &mgmt->schLock);
qError("taosHashPut sId[%"PRIx64"] to scheduleHash failed", sId);
QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno);
taosHashCleanup(newSch.tasksHash);
return TSDB_CODE_QRY_APP_ERROR;
}
}
QW_UNLOCK(QW_WRITE, &mgmt->schLock);
if (TSDB_CODE_SUCCESS == qwAcquireScheduler(rwType, mgmt, sId, sch, QW_NOT_EXIST_ADD)) {
if (TSDB_CODE_SUCCESS == qwAcquireScheduler(rwType, mgmt, sId, sch)) {
if (code) {
taosHashCleanup(newSch.tasksHash);
}
return TSDB_CODE_SUCCESS;
}
}
......@@ -141,7 +150,7 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId,
return TSDB_CODE_SUCCESS;
}
static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) {
int32_t qwAcquireSchedulerImpl(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) {
QW_LOCK(rwType, &mgmt->schLock);
*sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId));
if (NULL == (*sch)) {
......@@ -159,34 +168,19 @@ static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t s
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) {
QW_UNLOCK(rwType, &mgmt->schLock);
}
static int32_t qwAcquireTaskImpl(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) {
char id[sizeof(qId) + sizeof(tId)] = {0};
QW_SET_QTID(id, qId, tId);
QW_LOCK(rwType, &sch->tasksLock);
*task = taosHashGet(sch->tasksHash, id, sizeof(id));
if (NULL == (*task)) {
QW_UNLOCK(rwType, &sch->tasksLock);
return TSDB_CODE_QRY_TASK_NOT_EXIST;
}
return TSDB_CODE_SUCCESS;
int32_t qwAcquireAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) {
return qwAcquireSchedulerImpl(rwType, mgmt, sId, sch, QW_NOT_EXIST_ADD);
}
static int32_t qwAcquireTask(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) {
return qwAcquireTaskImpl(rwType, sch, qId, tId, task);
int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) {
return qwAcquireSchedulerImpl(rwType, mgmt, sId, sch, QW_NOT_EXIST_RET_ERR);
}
static FORCE_INLINE void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) {
QW_UNLOCK(rwType, &sch->tasksLock);
void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) {
QW_UNLOCK(rwType, &mgmt->schLock);
}
int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int8_t status, int32_t eOpt, SQWTaskStatus **task) {
int32_t qwAddTaskImpl(SQWorkerMgmt *mgmt, SQWSchStatus *sch, int32_t rwType, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWTaskStatus **task) {
int32_t code = 0;
char id[sizeof(qId) + sizeof(tId)] = {0};
......@@ -195,23 +189,18 @@ int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t
SQWTaskStatus ntask = {0};
ntask.status = status;
while (true) {
QW_LOCK(QW_WRITE, &sch->tasksLock);
int32_t code = taosHashPut(sch->tasksHash, id, sizeof(id), &ntask, sizeof(ntask));
code = taosHashPut(sch->tasksHash, id, sizeof(id), &ntask, sizeof(ntask));
if (0 != code) {
QW_UNLOCK(QW_WRITE, &sch->tasksLock);
if (HASH_NODE_EXIST(code)) {
if (QW_EXIST_ACQUIRE == eOpt && rwType && task) {
if (qwAcquireTask(rwType, sch, qId, tId, task)) {
continue;
}
QW_ERR_RET(qwAcquireTask(mgmt, rwType, sch, qId, tId, task));
} else if (QW_EXIST_RET_ERR == eOpt) {
return TSDB_CODE_QRY_TASK_ALREADY_EXIST;
} else {
assert(0);
}
break;
} else {
qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", qId, tId);
return TSDB_CODE_QRY_APP_ERROR;
......@@ -220,52 +209,78 @@ int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t
QW_UNLOCK(QW_WRITE, &sch->tasksLock);
if (rwType && task) {
if (TSDB_CODE_SUCCESS == qwAcquireTask(rwType, sch, qId, tId, task)) {
return TSDB_CODE_SUCCESS;
}
} else {
break;
}
if (QW_EXIST_ACQUIRE == eOpt && rwType && task) {
QW_ERR_RET(qwAcquireTask(mgmt, rwType, sch, qId, tId, task));
}
return TSDB_CODE_SUCCESS;
}
static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWSchStatus **sch, SQWTaskStatus **task) {
int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status) {
SQWSchStatus *tsch = NULL;
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &tsch, QW_NOT_EXIST_ADD));
int32_t code = 0;
QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &tsch));
int32_t code = qwAddTaskToSch(QW_READ, tsch, qId, tId, status, eOpt, task);
if (code) {
qwReleaseScheduler(QW_WRITE, mgmt);
}
QW_ERR_JRET(qwAddTaskImpl(mgmt, tsch, 0, qId, tId, status, QW_EXIST_RET_ERR, NULL));
_return:
if (NULL == task) {
qwReleaseScheduler(QW_READ, mgmt);
} else if (sch) {
*sch = tsch;
QW_ERR_RET(code);
}
int32_t qwAcquireTaskImpl(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int32_t status, int32_t nOpt, SQWTaskStatus **task) {
char id[sizeof(qId) + sizeof(tId)] = {0};
QW_SET_QTID(id, qId, tId);
QW_LOCK(rwType, &sch->tasksLock);
*task = taosHashGet(sch->tasksHash, id, sizeof(id));
if (NULL == (*task)) {
QW_UNLOCK(rwType, &sch->tasksLock);
if (QW_NOT_EXIST_ADD == nOpt) {
QW_ERR_RET(qwAddTaskImpl(mgmt, sch, rwType, qId, tId, status, QW_EXIST_ACQUIRE, task));
} else if (QW_NOT_EXIST_RET_ERR == nOpt) {
return TSDB_CODE_QRY_TASK_NOT_EXIST;
} else {
assert(0);
}
}
QW_RET(code);
return TSDB_CODE_SUCCESS;
}
int32_t qwAcquireTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) {
return qwAcquireTaskImpl(mgmt, rwType, sch, qId, tId, 0, QW_NOT_EXIST_RET_ERR, task);
}
int32_t qwAcquireAddTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int32_t status, SQWTaskStatus **task) {
return qwAcquireTaskImpl(mgmt, rwType, sch, qId, tId, status, QW_NOT_EXIST_ADD, task);
}
void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) {
QW_UNLOCK(rwType, &sch->tasksLock);
}
static FORCE_INLINE int32_t qwAcquireTaskHandles(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerTaskHandlesCache **handles) {
int32_t qwAcquireTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWTaskCtx **handles) {
char id[sizeof(queryId) + sizeof(taskId)] = {0};
QW_SET_QTID(id, queryId, taskId);
QW_LOCK(rwType, &mgmt->resLock);
*handles = taosHashGet(mgmt->resHash, id, sizeof(id));
QW_LOCK(rwType, &mgmt->ctxLock);
*handles = taosHashGet(mgmt->ctxHash, id, sizeof(id));
if (NULL == (*handles)) {
QW_UNLOCK(rwType, &mgmt->resLock);
QW_UNLOCK(rwType, &mgmt->ctxLock);
return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void qwReleaseTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt) {
QW_UNLOCK(rwType, &mgmt->resLock);
void qwReleaseTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt) {
QW_UNLOCK(rwType, &mgmt->ctxLock);
}
......@@ -273,7 +288,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs
SQWSchStatus *sch = NULL;
int32_t taskNum = 0;
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR));
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch));
sch->lastAccessTs = taosGetTimestampSec();
......@@ -319,7 +334,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs
int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId) {
SQWSchStatus *sch = NULL;
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR));
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch));
sch->lastAccessTs = taosGetTimestampSec();
......@@ -333,12 +348,12 @@ int32_t qwUpdateTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6
SQWTaskStatus *task = NULL;
int32_t code = 0;
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR));
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch));
QW_ERR_JRET(qwAcquireTask(QW_READ, sch, qId, tId, &task));
QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task));
QW_LOCK(QW_WRITE, &task->lock);
qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status);
qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS());
QW_UNLOCK(QW_WRITE, &task->lock);
_return:
......@@ -355,12 +370,12 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint
SQWTaskStatus *task = NULL;
int32_t code = 0;
if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) {
if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch)) {
*taskStatus = JOB_TASK_STATUS_NULL;
return TSDB_CODE_SUCCESS;
}
if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) {
if (qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)) {
qwReleaseScheduler(QW_READ, mgmt);
*taskStatus = JOB_TASK_STATUS_NULL;
......@@ -376,22 +391,15 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint
}
int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) {
int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD));
QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch));
if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) {
qwReleaseScheduler(QW_READ, mgmt);
QW_ERR_JRET(qwAcquireAddTask(mgmt, QW_READ, sch, qId, tId, JOB_TASK_STATUS_NOT_START, &task));
code = qwAddTask(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task);
if (code) {
qwReleaseScheduler(QW_READ, mgmt);
QW_ERR_RET(code);
}
}
QW_LOCK(QW_WRITE, &task->lock);
......@@ -409,10 +417,10 @@ int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_
return TSDB_CODE_SUCCESS;
} else if (task->status == JOB_TASK_STATUS_FAILED || task->status == JOB_TASK_STATUS_SUCCEED || task->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) {
newStatus = JOB_TASK_STATUS_CANCELLED;
QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus));
QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS()));
} else {
newStatus = JOB_TASK_STATUS_CANCELLING;
QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus));
QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS()));
}
QW_UNLOCK(QW_WRITE, &task->lock);
......@@ -441,55 +449,87 @@ _return:
QW_RET(code);
}
int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) {
// caller should make sure task is not running
int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) {
char id[sizeof(qId) + sizeof(tId)] = {0};
QW_SET_QTID(id, qId, tId);
QW_LOCK(QW_WRITE, &mgmt->ctxLock);
SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id));
if (NULL == ctx) {
QW_UNLOCK(QW_WRITE, &mgmt->ctxLock);
return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST;
}
if (ctx->taskHandle) {
qDestroyTask(ctx->taskHandle);
ctx->taskHandle = NULL;
}
if (ctx->sinkHandle) {
dsDestroyDataSinker(ctx->sinkHandle);
ctx->sinkHandle = NULL;
}
if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) {
QW_TASK_ELOG("taosHashRemove from ctx hash failed, id:%s", id);
QW_UNLOCK(QW_WRITE, &mgmt->ctxLock);
return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST;
}
QW_UNLOCK(QW_WRITE, &mgmt->ctxLock);
return TSDB_CODE_SUCCESS;
}
int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
char id[sizeof(queryId) + sizeof(taskId)] = {0};
QW_SET_QTID(id, queryId, taskId);
QW_LOCK(QW_WRITE, &mgmt->resLock);
if (mgmt->resHash) {
taosHashRemove(mgmt->resHash, id, sizeof(id));
}
QW_UNLOCK(QW_WRITE, &mgmt->resLock);
char id[sizeof(qId) + sizeof(tId)] = {0};
QW_SET_QTID(id, qId, tId);
qwDropTaskCtx(mgmt, sId, qId, tId);
if (TSDB_CODE_SUCCESS != qwAcquireScheduler(QW_WRITE, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) {
qWarn("scheduler %"PRIx64" doesn't exist", sId);
if (qwAcquireScheduler(QW_WRITE, mgmt, sId, &sch)) {
QW_TASK_WLOG("scheduler does not exist, sch:%p", sch);
return TSDB_CODE_SUCCESS;
}
if (qwAcquireTask(QW_WRITE, sch, queryId, taskId, &task)) {
if (qwAcquireTask(mgmt, QW_WRITE, sch, qId, tId, &task)) {
qwReleaseScheduler(QW_WRITE, mgmt);
qWarn("scheduler %"PRIx64" queryId %"PRIx64" taskId:%"PRIx64" doesn't exist", sId, queryId, taskId);
QW_TASK_WLOG("task does not exist, task:%p", task);
return TSDB_CODE_SUCCESS;
}
taosHashRemove(sch->tasksHash, id, sizeof(id));
QW_TASK_DLOG("drop task, status:%d, code:%x, ready:%d, cancel:%d, drop:%d", task->status, task->code, task->ready, task->cancel, task->drop);
if (taosHashRemove(sch->tasksHash, id, sizeof(id))) {
QW_TASK_ELOG("taosHashRemove task from hash failed, task:%p", task);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
_return:
qwReleaseTask(QW_WRITE, sch);
qwReleaseScheduler(QW_WRITE, mgmt);
return TSDB_CODE_SUCCESS;
QW_RET(code);
}
int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) {
int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD));
QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch));
if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) {
qwReleaseScheduler(QW_READ, mgmt);
code = qwAddTask(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task);
if (code) {
qwReleaseScheduler(QW_READ, mgmt);
QW_ERR_RET(code);
}
}
QW_ERR_JRET(qwAcquireAddTask(mgmt, QW_READ, sch, qId, tId, JOB_TASK_STATUS_NOT_START, &task));
QW_LOCK(QW_WRITE, &task->lock);
......@@ -500,7 +540,7 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uin
if (task->status == JOB_TASK_STATUS_EXECUTING) {
newStatus = JOB_TASK_STATUS_DROPPING;
QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus));
QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS()));
} else if (task->status == JOB_TASK_STATUS_CANCELLING || task->status == JOB_TASK_STATUS_DROPPING || task->status == JOB_TASK_STATUS_NOT_START) {
QW_UNLOCK(QW_WRITE, &task->lock);
qwReleaseTask(QW_READ, sch);
......@@ -512,11 +552,12 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uin
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
QW_ERR_RET(qwDropTask(mgmt, sId, queryId, taskId));
QW_ERR_RET(qwDropTask(mgmt, sId, qId, tId));
return TSDB_CODE_SUCCESS;
}
QW_UNLOCK(QW_WRITE, &task->lock);
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
......@@ -738,30 +779,35 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe
return TSDB_CODE_SUCCESS;
}
int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg, int32_t rspCode) {
int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR));
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch));
QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task));
QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task));
QW_LOCK(QW_WRITE, &task->lock);
if (QW_READY_NOT_RECEIVED == task->ready) {
QW_UNLOCK(QW_WRITE, &task->lock);
QW_SCH_TASK_DLOG("ready not received, ready:%d", task->ready);
goto _return;
} else if (QW_READY_RECEIVED == task->ready) {
task->ready = QW_READY_RESPONSED;
int32_t rspCode = task->code;
QW_UNLOCK(QW_WRITE, &task->lock);
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
return TSDB_CODE_SUCCESS;
} else if (QW_READY_RECEIVED == task->ready) {
QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, rspCode));
QW_ERR_RET(qwBuildAndSendReadyRsp(pMsg, rspCode));
task->ready = QW_READY_RESPONSED;
QW_SCH_TASK_DLOG("ready response sent, ready:%d", task->ready);
return TSDB_CODE_SUCCESS;
} else if (QW_READY_RESPONSED == task->ready) {
qError("query response already send");
QW_SCH_TASK_ELOG("ready response already send, ready:%d", task->ready);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} else {
assert(0);
......@@ -772,7 +818,6 @@ _return:
if (task) {
QW_UNLOCK(QW_WRITE, &task->lock);
qwReleaseTask(QW_READ, sch);
}
qwReleaseScheduler(QW_READ, mgmt);
......@@ -780,34 +825,39 @@ _return:
QW_RET(code);
}
int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) {
int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR));
QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch));
QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task));
QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task));
QW_LOCK(QW_WRITE, &task->lock);
if (QW_TASK_READY_RESP(task->status)) {
QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, task->code));
int8_t status = task->status;
int32_t errCode = task->code;
if (QW_TASK_READY(status)) {
task->ready = QW_READY_RESPONSED;
QW_UNLOCK(QW_WRITE, &task->lock);
QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, errCode));
QW_SCH_TASK_DLOG("task ready responsed, status:%d", status);
} else {
task->ready = QW_READY_RECEIVED;
QW_UNLOCK(QW_WRITE, &task->lock);
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
QW_UNLOCK(QW_WRITE, &task->lock);
return TSDB_CODE_SUCCESS;
QW_SCH_TASK_DLOG("task ready NOT responsed, status:%d", status);
}
_return:
if (task) {
QW_UNLOCK(QW_WRITE, &task->lock);
qwReleaseTask(QW_READ, sch);
}
......@@ -816,7 +866,7 @@ _return:
QW_RET(code);
}
int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, bool *needStop) {
int32_t qwCheckAndProcessTaskDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool *needStop) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
......@@ -824,44 +874,39 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI
*needStop = false;
if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) {
if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch)) {
return TSDB_CODE_SUCCESS;
}
if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) {
if (qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)) {
qwReleaseScheduler(QW_READ, mgmt);
return TSDB_CODE_SUCCESS;
}
QW_LOCK(QW_READ, &task->lock);
if ((!task->cancel) && (!task->drop)) {
qError("no cancel or drop, but task:%"PRIx64" exists", taskId);
QW_UNLOCK(QW_READ, &task->lock);
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
QW_RET(TSDB_CODE_QRY_APP_ERROR);
if ((!atomic_load_8(&task->cancel)) && (!atomic_load_8(&task->drop))) {
QW_TASK_ELOG("no cancel or drop but task exists, status:%d", atomic_load_8(&task->status));
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
QW_UNLOCK(QW_READ, &task->lock);
*needStop = true;
if (task->cancel) {
if (atomic_load_8(&task->cancel)) {
QW_LOCK(QW_WRITE, &task->lock);
qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status);
code = qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS());
QW_UNLOCK(QW_WRITE, &task->lock);
QW_ERR_JRET(code);
}
if (task->drop) {
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
return qwDropTask(mgmt, sId, queryId, taskId);
QW_RET(qwDropTask(mgmt, sId, qId, tId));
}
_return:
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
......@@ -875,31 +920,27 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6
int32_t code = 0;
int8_t newStatus = JOB_TASK_STATUS_CANCELLED;
code = qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD);
code = qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch);
if (code) {
qError("sId:%"PRIx64" not in cache", sId);
QW_TASK_ELOG("sId:%"PRIx64" not in cache", sId);
QW_ERR_RET(code);
}
code = qwAcquireTask(QW_READ, sch, qId, tId, &task);
code = qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task);
if (code) {
qwReleaseScheduler(QW_READ, mgmt);
if (JOB_TASK_STATUS_PARTIAL_SUCCEED == status || JOB_TASK_STATUS_SUCCEED == status) {
qError("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId);
QW_TASK_ELOG("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId);
QW_ERR_RET(code);
}
QW_ERR_RET(qwAddTask(mgmt, sId, qId, tId, status, QW_EXIST_ACQUIRE, &sch, &task));
}
QW_LOCK(QW_WRITE, &task->lock);
if (task->cancel) {
QW_LOCK(QW_WRITE, &task->lock);
qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus);
QW_UNLOCK(QW_WRITE, &task->lock);
qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS());
}
if (task->drop) {
QW_UNLOCK(QW_WRITE, &task->lock);
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
......@@ -909,20 +950,101 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6
}
if (!(task->cancel || task->drop)) {
QW_LOCK(QW_WRITE, &task->lock);
qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status);
qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS());
task->code = errCode;
QW_UNLOCK(QW_WRITE, &task->lock);
}
QW_UNLOCK(QW_WRITE, &task->lock);
qwReleaseTask(QW_READ, sch);
qwReleaseScheduler(QW_READ, mgmt);
return TSDB_CODE_SUCCESS;
}
int32_t qwScheduleDataSink(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) {
if (atomic_load_8(&handles->sinkScheduled)) {
qDebug("data sink already scheduled");
return TSDB_CODE_SUCCESS;
}
SSinkDataReq * req = (SSinkDataReq *)rpcMallocCont(sizeof(SSinkDataReq));
if (NULL == req) {
qError("rpcMallocCont %d failed", (int32_t)sizeof(SSinkDataReq));
QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
req->header.vgId = mgmt->nodeId;
req->sId = sId;
req->queryId = queryId;
req->taskId = taskId;
SRpcMsg pNewMsg = {
.handle = pMsg->handle,
.ahandle = pMsg->ahandle,
.msgType = TDMT_VND_SCHEDULE_DATA_SINK,
.pCont = req,
.contLen = sizeof(SSinkDataReq),
.code = 0,
};
int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg);
if (TSDB_CODE_SUCCESS != code) {
qError("put data sink schedule msg to queue failed, code:%x", code);
rpcFreeCont(req);
QW_ERR_RET(code);
}
qDebug("put data sink schedule msg to query queue");
return TSDB_CODE_SUCCESS;
}
int32_t qwScheduleQuery(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) {
if (atomic_load_8(&handles->queryScheduled)) {
QW_SCH_TASK_ELOG("query already scheduled, queryScheduled:%d", handles->queryScheduled);
return TSDB_CODE_SUCCESS;
}
QW_ERR_RET(qwUpdateTaskStatus(mgmt, sId, qId, tId, JOB_TASK_STATUS_EXECUTING));
int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) {
SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq));
if (NULL == req) {
QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq));
QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
req->header.vgId = mgmt->nodeId;
req->sId = sId;
req->queryId = qId;
req->taskId = tId;
SRpcMsg pNewMsg = {
.handle = pMsg->handle,
.ahandle = pMsg->ahandle,
.msgType = TDMT_VND_QUERY_CONTINUE,
.pCont = req,
.contLen = sizeof(SQueryContinueReq),
.code = 0,
};
int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg);
if (TSDB_CODE_SUCCESS != code) {
QW_SCH_TASK_ELOG("put query continue msg to queue failed, code:%x", code);
rpcFreeCont(req);
QW_ERR_RET(code);
}
handles->queryScheduled = true;
QW_SCH_TASK_DLOG("put query continue msg to query queue, vgId:%d", mgmt->nodeId);
return TSDB_CODE_SUCCESS;
}
int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
......@@ -932,22 +1054,30 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64
int32_t dataLength = 0;
SRetrieveTableRsp *rsp = NULL;
bool queryEnd = false;
SQWorkerTaskHandlesCache *handles = NULL;
SQWTaskCtx *handles = NULL;
int8_t status = 0;
QW_ERR_JRET(qwAcquireTaskHandles(QW_READ, mgmt, queryId, taskId, &handles));
QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, mgmt, qId, tId, &handles));
QW_LOCK(QW_WRITE, &handles->lock);
QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR));
QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task));
if (handles->needRsp) {
QW_UNLOCK(QW_WRITE, &handles->lock);
QW_SCH_TASK_ELOG("last fetch not responsed, needRsp:%d", handles->needRsp);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
QW_UNLOCK(QW_WRITE, &handles->lock);
QW_LOCK(QW_READ, &task->lock);
QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch));
QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task));
if (task->cancel || task->drop) {
qError("task is already cancelled or dropped");
QW_SCH_TASK_ELOG("task is already cancelled or dropped, cancel:%d, drop:%d", task->cancel, task->drop);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
qError("invalid status %d for fetch", task->status);
QW_SCH_TASK_ELOG("invalid status %d for fetch", task->status);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
......@@ -955,6 +1085,9 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64
if (dataLength > 0) {
SOutputData output = {0};
QW_SCH_TASK_DLOG("task got data in sink, dataLength:%d", dataLength);
QW_ERR_JRET(qwInitFetchRsp(dataLength, &rsp));
output.pData = rsp->data;
......@@ -974,29 +1107,39 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64
if (DS_BUF_EMPTY == output.bufStatus && output.queryEnd) {
rsp->completed = 1;
}
if (output.needSchedule) {
//TODO
status = JOB_TASK_STATUS_SUCCEED;
QW_SCH_TASK_DLOG("task all fetched, status:%d", status);
QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()));
}
if ((!output.queryEnd) && DS_BUF_LOW == output.bufStatus) {
//TODO
//UPDATE STATUS TO EXECUTING
// Note: schedule data sink firstly and will schedule query after it's done
if (output.needSchedule) {
QW_SCH_TASK_DLOG("sink need schedule, queryEnd:%d", output.queryEnd);
QW_ERR_JRET(qwScheduleDataSink(handles, mgmt, sId, qId, tId, pMsg));
} else if ((!output.queryEnd) && (DS_BUF_LOW == output.bufStatus || DS_BUF_EMPTY == output.bufStatus)) {
QW_SCH_TASK_DLOG("task not end, need to continue, bufStatus:%d", output.bufStatus);
QW_ERR_JRET(qwScheduleQuery(handles, mgmt, sId, qId, tId, pMsg));
}
} else {
if (dataLength < 0) {
qError("invalid length from dsGetDataLength, length:%d", dataLength);
QW_SCH_TASK_ELOG("invalid length from dsGetDataLength, length:%d", dataLength);
QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
if (queryEnd) {
QW_ERR_JRET(qwQueryPostProcess(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED, code));
status = JOB_TASK_STATUS_SUCCEED;
QW_SCH_TASK_DLOG("no data in sink and query end, dataLength:%d", dataLength);
QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()));
} else {
if (task->status != JOB_TASK_STATUS_EXECUTING) {
qError("invalid status %d for fetch without res", task->status);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
assert(0 == handles->needRsp);
// MUST IN SCHEDULE OR IN SINK SCHEDULE
QW_SCH_TASK_DLOG("no res data in sink, need response later, queryEnd:%d", queryEnd);
QW_LOCK(QW_WRITE, &handles->lock);
handles->needRsp = true;
......@@ -1009,7 +1152,6 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64
_return:
if (task) {
QW_UNLOCK(QW_READ, &task->lock);
qwReleaseTask(QW_READ, sch);
}
......@@ -1028,7 +1170,12 @@ _return:
QW_RET(code);
}
int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) {
int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp) {
if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp) {
qError("invalid param to init qworker");
QW_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt));
if (NULL == mgmt) {
qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt));
......@@ -1037,29 +1184,46 @@ int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) {
if (cfg) {
mgmt->cfg = *cfg;
if (0 == mgmt->cfg.maxSchedulerNum) {
mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER;
}
if (0 == mgmt->cfg.maxTaskNum) {
mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER;
}
if (0 == mgmt->cfg.maxSchTaskNum) {
mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER;
}
} else {
mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER;
mgmt->cfg.maxResCacheNum = QWORKER_DEFAULT_RES_CACHE_NUMBER;
mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER;
mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER;
}
mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
if (NULL == mgmt->schHash) {
tfree(mgmt);
QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler hash failed", mgmt->cfg.maxSchedulerNum);
qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum);
QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
mgmt->resHash = taosHashInit(mgmt->cfg.maxResCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (NULL == mgmt->resHash) {
mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (NULL == mgmt->ctxHash) {
taosHashCleanup(mgmt->schHash);
mgmt->schHash = NULL;
tfree(mgmt);
QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d res cache hash failed", mgmt->cfg.maxResCacheNum);
qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum);
QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
mgmt->nodeType = nodeType;
mgmt->nodeId = nodeId;
mgmt->nodeObj = nodeObj;
mgmt->putToQueueFp = fp;
*qWorkerMgmt = mgmt;
qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt);
return TSDB_CODE_SUCCESS;
}
......@@ -1069,40 +1233,47 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
}
int32_t code = 0;
bool queryRsped = false;
bool needStop = false;
struct SSubplan *plan = NULL;
SSubQueryMsg *msg = pMsg->pCont;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
int32_t rspCode = 0;
if (NULL == msg || pMsg->contLen <= sizeof(*msg)) {
qError("invalid query msg");
QW_ELOG("invalid query msg, contLen:%d", pMsg->contLen);
QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
msg->sId = htobe64(msg->sId);
msg->queryId = htobe64(msg->queryId);
msg->taskId = htobe64(msg->taskId);
msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId);
msg->taskId = be64toh(msg->taskId);
msg->contentLen = ntohl(msg->contentLen);
bool queryRsped = false;
bool needStop = false;
struct SSubplan *plan = NULL;
uint64_t sId = msg->sId;
uint64_t qId = msg->queryId;
uint64_t tId = msg->taskId;
QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop));
QW_ERR_JRET(qwCheckAndProcessTaskDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop));
if (needStop) {
qWarn("task need stop");
QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED);
QW_TASK_DLOG("task need stop, msgLen:%d", msg->contentLen);
qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED);
QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED);
}
QW_ERR_JRET(qwAddTask(qWorkerMgmt, sId, qId, tId, JOB_TASK_STATUS_EXECUTING));
code = qStringToSubplan(msg->msg, &plan);
if (TSDB_CODE_SUCCESS != code) {
qError("schId:%"PRIx64",qId:%"PRIx64",taskId:%"PRIx64" string to subplan failed, code:%d", msg->sId, msg->queryId, msg->taskId, code);
QW_TASK_ELOG("string to subplan failed, code:%d", code);
QW_ERR_JRET(code);
}
qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo);
if (code) {
qError("qCreateExecTask failed, code:%x", code);
QW_TASK_ELOG("qCreateExecTask failed, code:%x", code);
QW_ERR_JRET(code);
} else {
QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL));
}
QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS));
......@@ -1111,63 +1282,115 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
DataSinkHandle sinkHandle = NULL;
code = qExecTask(pTaskInfo, &sinkHandle);
if (code) {
qError("qExecTask failed, code:%x", code);
QW_TASK_ELOG("qExecTask failed, code:%x", code);
QW_ERR_JRET(code);
} else {
QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle));
QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED));
}
QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle));
_return:
if (queryRsped) {
code = qwCheckAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg, code);
} else {
code = qwBuildAndSendQueryRsp(pMsg, code);
if (code) {
rspCode = code;
}
if (!queryRsped) {
code = qwBuildAndSendQueryRsp(pMsg, rspCode);
if (TSDB_CODE_SUCCESS == rspCode && code) {
rspCode = code;
}
}
int8_t status = 0;
if (TSDB_CODE_SUCCESS != code) {
if (TSDB_CODE_SUCCESS != rspCode) {
status = JOB_TASK_STATUS_FAILED;
} else {
status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
}
qwQueryPostProcess(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, status, code);
qwQueryPostProcess(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, status, rspCode);
QW_RET(code);
if (queryRsped) {
qwCheckAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg);
}
QW_RET(rspCode);
}
int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
int8_t status = 0;
bool queryDone = false;
uint64_t sId, qId, tId;
SQueryContinueReq *req = (SQueryContinueReq *)pMsg->pCont;
bool needStop = false;
SQWTaskCtx *handles = NULL;
//TODO call executer to continue execute subquery
code = 0;
void *data = NULL;
queryDone = false;
//TODO call executer to continue execute subquery
QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles));
QW_LOCK(QW_WRITE, &handles->lock);
qTaskInfo_t taskHandle = handles->taskHandle;
DataSinkHandle sinkHandle = handles->sinkHandle;
QW_UNLOCK(QW_WRITE, &handles->lock);
qwReleaseTaskResCache(QW_READ, qWorkerMgmt);
QW_ERR_JRET(qwCheckAndProcessTaskDrop(qWorkerMgmt, req->sId, req->queryId, req->taskId, &needStop));
if (needStop) {
qWarn("task need stop");
QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles));
QW_LOCK(QW_WRITE, &handles->lock);
if (handles->needRsp) {
qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED);
handles->needRsp = false;
}
QW_UNLOCK(QW_WRITE, &handles->lock);
qwReleaseTaskResCache(QW_READ, qWorkerMgmt);
QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED);
}
DataSinkHandle newHandle = NULL;
code = qExecTask(taskHandle, &newHandle);
if (code) {
qError("qExecTask failed, code:%x", code);
QW_ERR_JRET(code);
}
if (sinkHandle != newHandle) {
qError("data sink mis-match");
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
_return:
QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles));
QW_LOCK(QW_WRITE, &handles->lock);
if (handles->needRsp) {
code = qwBuildAndSendQueryRsp(pMsg, code);
handles->needRsp = false;
}
handles->queryScheduled = false;
QW_UNLOCK(QW_WRITE, &handles->lock);
qwReleaseTaskResCache(QW_READ, qWorkerMgmt);
if (TSDB_CODE_SUCCESS != code) {
status = JOB_TASK_STATUS_FAILED;
} else if (queryDone) {
status = JOB_TASK_STATUS_SUCCEED;
} else {
status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
}
code = qwQueryPostProcess(qWorkerMgmt, sId, qId, tId, status, code);
code = qwQueryPostProcess(qWorkerMgmt, req->sId, req->queryId, req->taskId, status, code);
QW_RET(code);
}
int32_t qWorkerProcessSinkDataMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
......@@ -1178,6 +1401,7 @@ int32_t qWorkerProcessSinkDataMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
//dsScheduleProcess();
//TODO
return TSDB_CODE_SUCCESS;
......
source/libs/qworker/test/qworkerTests.cpp
浏览文件 @ ae2d77e9
......@@ -42,6 +42,11 @@ int32_t qwtStringToPlan(const char* str, SSubplan** subplan) {
return 0;
}
int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) {
return 0;
}
void qwtRpcSendResponse(const SRpcMsg *pRsp) {
if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) {
SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont;
......@@ -258,7 +263,7 @@ TEST(seqTest, normalCase) {
stubSetStringToPlan();
stubSetRpcSendResponse();
code = qWorkerInit(NULL, &mgmt);
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
......@@ -328,7 +333,7 @@ TEST(seqTest, cancelFirst) {
stubSetStringToPlan();
stubSetRpcSendResponse();
code = qWorkerInit(NULL, &mgmt);
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
......@@ -402,7 +407,7 @@ TEST(seqTest, randCase) {
srand(time(NULL));
code = qWorkerInit(NULL, &mgmt);
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
ASSERT_EQ(code, 0);
int32_t t = 0;
......@@ -446,7 +451,7 @@ TEST(seqTest, multithreadRand) {
srand(time(NULL));
code = qWorkerInit(NULL, &mgmt);
code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
ASSERT_EQ(code, 0);
pthread_attr_t thattr;
......
source/libs/scheduler/inc/schedulerInt.h
浏览文件 @ ae2d77e9
......@@ -36,11 +36,31 @@ enum {
SCH_WRITE,
};
typedef struct SSchApiStat {
} SSchApiStat;
typedef struct SSchRuntimeStat {
} SSchRuntimeStat;
typedef struct SSchJobStat {
} SSchJobStat;
typedef struct SSchedulerStat {
SSchApiStat api;
SSchRuntimeStat runtime;
SSchJobStat job;
} SSchedulerStat;
typedef struct SSchedulerMgmt {
uint64_t taskId; // sequential taksId
uint64_t sId; // schedulerId
SSchedulerCfg cfg;
SHashObj *jobs; // key: queryId, value: SQueryJob*
SSchedulerStat stat;
} SSchedulerMgmt;
typedef struct SSchCallbackParam {
......
source/libs/scheduler/src/scheduler.c
浏览文件 @ ae2d77e9
......@@ -50,6 +50,10 @@ void schFreeTask(SSchTask* pTask) {
if (pTask->parents) {
taosArrayDestroy(pTask->parents);
}
if (pTask->execAddrs) {
taosArrayDestroy(pTask->execAddrs);
}
}
......@@ -798,7 +802,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
SQueryTableRsp *rsp = (SQueryTableRsp *)msg;
if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) {
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code));
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode));
}
SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY));
......@@ -1025,6 +1029,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
msg = pTask->msg;
break;
}
case TDMT_VND_QUERY: {
msgSize = sizeof(SSubQueryMsg) + pTask->msgLen;
msg = calloc(1, msgSize);
......@@ -1044,6 +1049,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
memcpy(pMsg->msg, pTask->msg, pTask->msgLen);
break;
}
case TDMT_VND_RES_READY: {
msgSize = sizeof(SResReadyReq);
msg = calloc(1, msgSize);
......@@ -1364,16 +1370,10 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag,
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SSchJob *job = NULL;
SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, false));
*pJob = job;
SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, false));
return TSDB_CODE_SUCCESS;
}
int32_t scheduleFetchRows(SSchJob *pJob, void** pData) {
if (NULL == pJob || NULL == pData) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
......@@ -1462,7 +1462,9 @@ void scheduleFreeJob(void *job) {
}
SSchJob *pJob = job;
uint64_t queryId = pJob->queryId;
if (SCH_GET_JOB_STATUS(pJob) > 0) {
if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) {
SCH_JOB_ELOG("taosHashRemove job from list failed, may already freed, pJob:%p", pJob);
return;
......@@ -1490,6 +1492,7 @@ void scheduleFreeJob(void *job) {
}
schDropJobAllTasks(pJob);
}
pJob->subPlans = NULL; // it is a reference to pDag->pSubplans
......@@ -1515,6 +1518,8 @@ void scheduleFreeJob(void *job) {
tfree(pJob->res);
tfree(pJob);
qDebug("QID:%"PRIx64" job freed", queryId);
}
void schedulerDestroy(void) {
......
source/libs/scheduler/test/schedulerTests.cpp
浏览文件 @ ae2d77e9
......@@ -79,6 +79,7 @@ void schtBuildQueryDag(SQueryDag *dag) {
scanPlan->level = 1;
scanPlan->pParents = taosArrayInit(1, POINTER_BYTES);
scanPlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode));
scanPlan->msgType = TDMT_VND_QUERY;
mergePlan->id.queryId = qId;
mergePlan->id.templateId = 0x4444444444;
......@@ -89,6 +90,7 @@ void schtBuildQueryDag(SQueryDag *dag) {
mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES);
mergePlan->pParents = NULL;
mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode));
mergePlan->msgType = TDMT_VND_QUERY;
SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan);
SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan);
......@@ -163,6 +165,11 @@ void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) {
}
void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {
}
void schtSetPlanToString() {
static Stub stub;
......@@ -190,6 +197,20 @@ void schtSetExecNode() {
}
}
void schtSetRpcSendRequest() {
static Stub stub;
stub.set(rpcSendRequest, schtRpcSendRequest);
{
AddrAny any("libtransport.so");
std::map<std::string,void*> result;
any.get_global_func_addr_dynsym("^rpcSendRequest$", result);
for (const auto& f : result) {
stub.set(f.second, schtRpcSendRequest);
}
}
}
void *schtSendRsp(void *param) {
SSchJob *job = NULL;
int32_t code = 0;
......@@ -218,8 +239,6 @@ void *schtSendRsp(void *param) {
}
struct SSchJob *pInsertJob = NULL;
}
TEST(queryTest, normalCase) {
......
source/dnode/vnode/tq/CMakeLists.txt → source/libs/tdb/CMakeLists.txt
浏览文件 @ ae2d77e9
aux_source_directory(src TQ_SRC)
add_library(tq ${TQ_SRC})
set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr")
foreach(TDB_SUBDIR ${TDB_SUBDIRS})
aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC)
endforeach()
add_library(tdb STATIC ${TDB_SRC})
target_include_directories(
tq
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tq"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
tdb
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src/inc"
)
target_link_libraries(
tq
PUBLIC wal
tdb
PUBLIC os
PUBLIC util
PUBLIC common
PUBLIC transport
)
if(${BUILD_TEST})
......
source/libs/tdb/inc/tdb.h 0 → 100644
浏览文件 @ ae2d77e9
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_H_
#define _TD_TDB_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
#define TDB_EXTERN
#define TDB_PUBLIC
#define TDB_STATIC static
typedef enum {
TDB_BTREE_T = 0,
TDB_HASH_T,
TDB_HEAP_T,
} tdb_db_t;
// Forward declaration
typedef struct TDB TDB;
typedef struct TDB_CURSOR TDB_CURSOR;
// SKey
typedef struct {
void* bdata;
uint32_t size;
} TDB_KEY, TDB_VALUE;
// TDB Operations
TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type);
TDB_EXTERN int tdbOpenDB(TDB* dbp, uint32_t flags);
TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_H_*/
\ No newline at end of file
source/libs/tdb/src/db/tdbDB.c 0 → 100644
浏览文件 @ ae2d77e9
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbDB.h"
#include "tdb.h"
TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) {
TDB* dbp;
int ret;
dbp = calloc(1, sizeof(*dbp));
if (dbp == NULL) {
return -1;
}
dbp->pageSize = TDB_DEFAULT_PGSIZE;
dbp->type = type;
switch (type) {
case TDB_BTREE_T:
// ret = tdbInitBtreeDB(dbp);
// if (ret < 0) goto _err;
break;
case TDB_HASH_T:
// ret = tdbInitHashDB(dbp);
// if (ret < 0) goto _err;
break;
case TDB_HEAP_T:
// ret = tdbInitHeapDB(dbp);
// if (ret < 0) goto _err;
break;
default:
break;
}
*dbpp = dbp;
return 0;
_err:
if (dbp) {
free(dbp);
}
*dbpp = NULL;
return 0;
}
TDB_EXTERN int tdbOpenDB(TDB* dbp, uint32_t flags) {
// TODO
return 0;
}
TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) {
// TODO
return 0;
}
\ No newline at end of file
source/libs/tkv/src/tDiskMgr.c → source/libs/tdb/src/dmgr/tdbDiskMgr.c
浏览文件 @ ae2d77e9
......@@ -13,37 +13,62 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tDiskMgr.h"
#include "tdbDiskMgr.h"
struct SDiskMgr {
const char *fname;
struct STkvDiskMgr {
char * fname;
uint16_t pgsize;
FileFd fd;
int32_t npgid;
pgid_t npgid;
};
#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE))
int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) {
// TODO
int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) {
STkvDiskMgr *pDiskMgr;
pDiskMgr = malloc(sizeof(*pDiskMgr));
if (pDiskMgr == NULL) {
return -1;
}
pDiskMgr->fname = strdup(fname);
if (pDiskMgr->fname == NULL) {
free(pDiskMgr);
return -1;
}
pDiskMgr->pgsize = pgsize;
pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755);
if (pDiskMgr->fd < 0) {
free(pDiskMgr->fname);
free(pDiskMgr);
return -1;
}
*ppDiskMgr = pDiskMgr;
return 0;
}
int tdmClose(SDiskMgr *pDiskMgr) {
// TODO
int tdmClose(STkvDiskMgr *pDiskMgr) {
close(pDiskMgr->fd);
free(pDiskMgr->fname);
free(pDiskMgr);
return 0;
}
int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData) {
int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) {
taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET);
taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize);
return 0;
}
int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) {
int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) {
taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET);
taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize);
return 0;
}
int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; }
\ No newline at end of file
int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); }
int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; }
\ No newline at end of file
source/libs/tdb/src/inc/tdbBtree.h 0 → 100644
浏览文件 @ ae2d77e9
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_BTREE_H_
#define _TD_TDB_BTREE_H_
#include "tdbDef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
pgid_t root; // root page number
} TDB_BTREE;
TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_BTREE_H_*/
\ No newline at end of file
source/dnode/vnode/impl/inc/vnodeFS.h → source/libs/tdb/src/inc/tdbBufPool.h
浏览文件 @ ae2d77e9
......@@ -13,35 +13,27 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_FS_H_
#define _TD_VNODE_FS_H_
#ifndef _TD_TDB_BUF_POOL_H_
#define _TD_TDB_BUF_POOL_H_
#include "vnode.h"
#include "tdbPage.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
} SDir;
typedef struct STdbBufPool STdbBufPool;
typedef struct {
} SFile;
typedef struct SFS {
void *pImpl;
int (*startEdit)(struct SFS *);
int (*endEdit)(struct SFS *);
} SFS;
typedef struct {
} SVnodeFS;
int vnodeOpenFS(SVnode *pVnode);
void vnodeCloseFS(SVnode *pVnode);
int tbpOpen(STdbBufPool **ppTkvBufPool);
int tbpClose(STdbBufPool *pTkvBufPool);
STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool);
int tbpDelPage(STdbBufPool *pTkvBufPool);
STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid);
int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid);
void tbpFlushPages(STdbBufPool *pTkvBufPool);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_FS_H_*/
\ No newline at end of file
#endif /*_TD_TDB_BUF_POOL_H_*/
\ No newline at end of file
source/libs/tdb/src/inc/tdbDB.h 0 → 100644
浏览文件 @ ae2d77e9
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_DB_H_
#define _TD_TDB_DB_H_
#include "tdb.h"
#include "tdbBtree.h"
#include "tdbHash.h"
#include "tdbHeap.h"
#ifdef __cplusplus
extern "C" {
#endif
struct TDB {
pgsize_t pageSize;
tdb_db_t type;
union {
TDB_BTREE *btree;
TDB_HASH * hash;
TDB_HEAP * heap;
} dbam; // Different access methods
};
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_DB_H_*/
\ No newline at end of file
source/libs/tkv/inc/tkvDef.h → source/libs/tdb/src/inc/tdbDef.h
浏览文件 @ ae2d77e9
......@@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TKV_DEF_H_
#define _TD_TKV_DEF_H_
#ifndef _TD_TDB_DEF_H_
#define _TD_TDB_DEF_H_
#include "os.h"
......@@ -24,10 +24,20 @@ extern "C" {
// pgid_t
typedef int32_t pgid_t;
#define TKV_IVLD_PGID ((pgid_t)-1)
#define TDB_IVLD_PGID ((pgid_t)-1)
// framd_id_t
typedef int32_t frame_id_t;
// pgsize_t
typedef int32_t pgsize_t;
#define TDB_MIN_PGSIZE 512
#define TDB_MAX_PGSIZE 16384
#define TDB_DEFAULT_PGSIZE 4096
#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE))
#ifdef __cplusplus
}
#endif
#endif /*_TD_TKV_DEF_H_*/
\ No newline at end of file
#endif /*_TD_TDB_DEF_H_*/
\ No newline at end of file
source/libs/tkv/inc/tDiskMgr.h → source/libs/tdb/src/inc/tdbDiskMgr.h
浏览文件 @ ae2d77e9
......@@ -22,15 +22,16 @@ extern "C" {
#include "os.h"
#include "tkvDef.h"
#include "tdbDef.h"
typedef struct SDiskMgr SDiskMgr;
typedef struct STkvDiskMgr STkvDiskMgr;
int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize);
int tdmClose(SDiskMgr *pDiskMgr);
int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData);
int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData);
int32_t tdmAllocPage(SDiskMgr *pDiskMgr);
int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize);
int tdmClose(STkvDiskMgr *pDiskMgr);
int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData);
int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData);
int tdmFlush(STkvDiskMgr *pDiskMgr);
pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr);
#ifdef __cplusplus
}
......
source/dnode/vnode/impl/inc/vnodeSync.h → source/libs/tdb/src/inc/tdbHash.h
浏览文件 @ ae2d77e9
......@@ -13,21 +13,23 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_SYNC_H_
#define _TD_VNODE_SYNC_H_
#ifndef _TD_TDB_HASH_H_
#define _TD_TDB_HASH_H_
#include "sync.h"
#include "tdbDef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
/* data */
} SVnodeSync;
// TODO
} TDB_HASH;
TDB_PUBLIC int tdbInitHashDB(TDB *dbp);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_SYNC_H_*/
\ No newline at end of file
#endif /*_TD_TDB_HASH_H_*/
\ No newline at end of file
source/dnode/vnode/impl/inc/vnodeMAF.h → source/libs/tdb/src/inc/tdbHeap.h
浏览文件 @ ae2d77e9
......@@ -13,20 +13,23 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_MAF_H_
#define _TD_VNODE_MAF_H_
#ifndef _TD_TDB_HEAP_H_
#define _TD_TDB_HEAP_H_
#include "vnode.h"
#include "tdbDef.h"
#ifdef __cplusplus
extern "C" {
#endif
int vnodeOpenMAF(SVnode *pVnode);
void vnodeCloseMAF(SVnode *pVnode);
typedef struct {
// TODO
} TDB_HEAP;
TDB_PUBLIC int tdbInitHeapDB(TDB *dbp);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_MAF_H_*/
\ No newline at end of file
#endif /*_TD_TDB_HEAP_H_*/
\ No newline at end of file
source/libs/tkv/inc/tPage.h → source/libs/tdb/src/inc/tdbPage.h
浏览文件 @ ae2d77e9
......@@ -17,22 +17,24 @@
#define _TD_TKV_PAGE_H_
#include "os.h"
#include "tdbDef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
pgid_t pgid;
int32_t pinCount;
bool idDirty;
char* pData;
} STdbPage;
typedef struct {
uint16_t dbver;
uint16_t pgsize;
uint32_t cksm;
} SPgHdr;
typedef struct {
SPgHdr chdr;
uint16_t used; // number of used slots
uint16_t loffset; // the offset of the starting location of the last slot used
} SSlottedPgHdr;
} STdbPgHdr;
#ifdef __cplusplus
}
......
source/libs/tdb/src/mpool/tdbBufPool.c 0 → 100644
浏览文件 @ ae2d77e9
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "thash.h"
#include "tlist.h"
#include "tdbBufPool.h"
#include "tdbDiskMgr.h"
#include "tdbPage.h"
struct SFrameIdWrapper {
TD_SLIST_NODE(SFrameIdWrapper);
frame_id_t id;
};
struct STdbBufPool {
STdbPage* pages;
STkvDiskMgr* pDiskMgr;
SHashObj* pgTb; // page_id_t --> frame_id_t
TD_SLIST(SFrameIdWrapper) freeList;
pthread_mutex_t mutex;
};
typedef struct STkvLRUReplacer {
} STkvLRUReplacer;
typedef struct STkvLFUReplacer {
} STkvLFUReplacer;
typedef struct STkvCLKReplacer {
} STkvCLKReplacer;
typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t;
typedef struct STkvReplacer {
tkv_replacer_t type;
union {
STkvLRUReplacer lruRep;
STkvLFUReplacer lfuRep;
STkvCLKReplacer clkRep;
};
} STkvReplacer;
\ No newline at end of file
source/libs/tdb/test/CMakeLists.txt 0 → 100644
浏览文件 @ ae2d77e9
# tdbTest
add_executable(tdbTest "tdbTest.cpp")
target_link_libraries(tdbTest tdb gtest gtest_main)
\ No newline at end of file
source/libs/tdb/test/tdbTest.cpp 0 → 100644
浏览文件 @ ae2d77e9
#include "gtest/gtest.h"
#include "tdb.h"
TEST(tdb_api_test, tdb_create_open_close_db_test) {
int ret;
TDB *dbp;
tdbCreateDB(&dbp, TDB_BTREE_T);
tdbOpenDB(dbp, 0);
tdbCloseDB(dbp, 0);
}
\ No newline at end of file
source/libs/tkv/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ de8b6e73
aux_source_directory(src TKV_SRC)
add_library(tkv STATIC ${TKV_SRC})
target_include_directories(
tkv
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
tkv
PUBLIC os
)
\ No newline at end of file
source/libs/tkv/test/tDiskMgrTest.cpp 已删除 100644 → 0
浏览文件 @ de8b6e73
#include "gtest/gtest.h"
#include "iostream"
#include "tDiskMgr.h"
TEST(tDiskMgrTest, simple_test) {
// TODO
std::cout << "This is in tDiskMgrTest::simple_test" << std::endl;
}
\ No newline at end of file
source/libs/transport/CMakeLists.txt
浏览文件 @ ae2d77e9
......@@ -13,3 +13,25 @@ target_link_libraries(
PUBLIC util
PUBLIC common
)
if (${BUILD_WITH_UV})
target_include_directories(
transport
PUBLIC "${CMAKE_SOURCE_DIR}/contrib/libuv/include"
)
#LINK_DIRECTORIES("${CMAKE_SOURCE_DIR}/debug/contrib/libuv")
target_link_libraries(
transport
PUBLIC uv_a
)
add_definitions(-DUSE_UV)
endif(${BUILD_WITH_UV})
if (${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})
source/libs/transport/inc/rpcCache.h
浏览文件 @ ae2d77e9
......@@ -16,6 +16,8 @@
#ifndef TDENGINE_RPC_CACHE_H
#define TDENGINE_RPC_CACHE_H
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
......
source/libs/transport/inc/rpcHead.h
浏览文件 @ ae2d77e9
......@@ -16,6 +16,7 @@
#ifndef TDENGINE_RPCHEAD_H
#define TDENGINE_RPCHEAD_H
#include <tdef.h>
#ifdef __cplusplus
extern "C" {
#endif
......@@ -25,24 +26,24 @@ extern "C" {
extern int tsRpcOverhead;
typedef struct {
void *msg;
void* msg;
int msgLen;
uint32_t ip;
uint16_t port;
int connType;
void *shandle;
void *thandle;
void *chandle;
void* shandle;
void* thandle;
void* chandle;
} SRecvInfo;
#pragma pack(push, 1)
typedef struct {
char version:4; // RPC version
char comp:4; // compression algorithm, 0:no compression 1:lz4
char resflag:2; // reserved bits
char spi:3; // security parameter index
char encrypt:3; // encrypt algorithm, 0: no encryption
char version : 4; // RPC version
char comp : 4; // compression algorithm, 0:no compression 1:lz4
char resflag : 2; // reserved bits
char spi : 3; // security parameter index
char encrypt : 3; // encrypt algorithm, 0: no encryption
uint16_t tranId; // transcation ID
uint32_t linkUid; // for unique connection ID assigned by client
uint64_t ahandle; // ahandle assigned by client
......@@ -71,10 +72,8 @@ typedef struct {
#pragma pack(pop)
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_RPCHEAD_H
source/libs/transport/inc/rpcTcp.h
浏览文件 @ ae2d77e9
......@@ -15,6 +15,7 @@
#ifndef _rpc_tcp_header_
#define _rpc_tcp_header_
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
......
source/libs/transport/inc/transportInt.h
浏览文件 @ ae2d77e9
......@@ -16,10 +16,65 @@
#ifndef _TD_TRANSPORT_INT_H_
#define _TD_TRANSPORT_INT_H_
#include "rpcHead.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifdef USE_UV
#include <stddef.h>
typedef void *queue[2];
/* Private macros. */
#define QUEUE_NEXT(q) (*(queue **)&((*(q))[0]))
#define QUEUE_PREV(q) (*(queue **)&((*(q))[1]))
#define QUEUE_PREV_NEXT(q) (QUEUE_NEXT(QUEUE_PREV(q)))
#define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q)))
/* Initialize an empty queue. */
#define QUEUE_INIT(q) \
{ \
QUEUE_NEXT(q) = (q); \
QUEUE_PREV(q) = (q); \
}
/* Return true if the queue has no element. */
#define QUEUE_IS_EMPTY(q) ((const queue *)(q) == (const queue *)QUEUE_NEXT(q))
/* Insert an element at the back of a queue. */
#define QUEUE_PUSH(q, e) \
{ \
QUEUE_NEXT(e) = (q); \
QUEUE_PREV(e) = QUEUE_PREV(q); \
QUEUE_PREV_NEXT(e) = (e); \
QUEUE_PREV(q) = (e); \
}
/* Remove the given element from the queue. Any element can be removed at any *
* time. */
#define QUEUE_REMOVE(e) \
{ \
QUEUE_PREV_NEXT(e) = QUEUE_NEXT(e); \
QUEUE_NEXT_PREV(e) = QUEUE_PREV(e); \
}
/* Return the element at the front of the queue. */
#define QUEUE_HEAD(q) (QUEUE_NEXT(q))
/* Return the element at the back of the queue. */
#define QUEUE_TAIL(q) (QUEUE_PREV(q))
/* Iterate over the element of a queue. * Mutating the queue while iterating
* results in undefined behavior. */
#define QUEUE_FOREACH(q, e) for ((q) = QUEUE_NEXT(e); (q) != (e); (q) = QUEUE_NEXT(q))
/* Return the structure holding the given element. */
#define QUEUE_DATA(e, type, field) ((type *)((void *)((char *)(e)-offsetof(type, field))))
#endif // USE_LIBUV
#ifdef __cplusplus
}
#endif
......
source/libs/transport/src/rpcCache.c
浏览文件 @ ae2d77e9
......@@ -13,14 +13,14 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "rpcCache.h"
#include "os.h"
#include "rpcLog.h"
#include "taosdef.h"
#include "tglobal.h"
#include "tmempool.h"
#include "ttimer.h"
#include "tutil.h"
#include "rpcLog.h"
#include "rpcCache.h"
typedef struct SConnHash {
char fqdn[TSDB_FQDN_LEN];
......@@ -28,12 +28,12 @@ typedef struct SConnHash {
char connType;
struct SConnHash *prev;
struct SConnHash *next;
void *data;
void * data;
uint64_t time;
} SConnHash;
typedef struct {
SConnHash **connHashList;
SConnHash ** connHashList;
mpool_h connHashMemPool;
int maxSessions;
int total;
......@@ -41,8 +41,8 @@ typedef struct {
int64_t keepTimer;
pthread_mutex_t mutex;
void (*cleanFp)(void *);
void *tmrCtrl;
void *pTimer;
void * tmrCtrl;
void * pTimer;
int64_t *lockedBy;
} SConnCache;
......@@ -134,7 +134,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in
pNode->prev = NULL;
pNode->time = time;
rpcLockCache(pCache->lockedBy+hash);
rpcLockCache(pCache->lockedBy + hash);
pNode->next = pCache->connHashList[hash];
if (pCache->connHashList[hash] != NULL) (pCache->connHashList[hash])->prev = pNode;
......@@ -143,10 +143,11 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in
pCache->count[hash]++;
rpcRemoveExpiredNodes(pCache, pNode->next, hash, time);
rpcUnlockCache(pCache->lockedBy+hash);
rpcUnlockCache(pCache->lockedBy + hash);
pCache->total++;
// tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]);
// tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode,
// pCache->count[hash]);
return;
}
......@@ -163,7 +164,7 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy
uint64_t time = taosGetTimestampMs();
hash = rpcHashConn(pCache, fqdn, port, connType);
rpcLockCache(pCache->lockedBy+hash);
rpcLockCache(pCache->lockedBy + hash);
pNode = pCache->connHashList[hash];
while (pNode) {
......@@ -197,12 +198,14 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy
pCache->count[hash]--;
}
rpcUnlockCache(pCache->lockedBy+hash);
rpcUnlockCache(pCache->lockedBy + hash);
if (pData) {
//tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]);
// tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode,
// pCache->count[hash]);
} else {
//tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]);
// tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash,
// pCache->count[hash]);
}
return pData;
......@@ -221,10 +224,10 @@ static void rpcCleanConnCache(void *handle, void *tmrId) {
uint64_t time = taosGetTimestampMs();
for (hash = 0; hash < pCache->maxSessions; ++hash) {
rpcLockCache(pCache->lockedBy+hash);
rpcLockCache(pCache->lockedBy + hash);
pNode = pCache->connHashList[hash];
rpcRemoveExpiredNodes(pCache, pNode, hash, time);
rpcUnlockCache(pCache->lockedBy+hash);
rpcUnlockCache(pCache->lockedBy + hash);
}
// tTrace("timer, total connections in cache:%d", pCache->total);
......@@ -233,7 +236,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) {
}
static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) {
if (pNode == NULL || (time < pCache->keepTimer + pNode->time) ) return;
if (pNode == NULL || (time < pCache->keepTimer + pNode->time)) return;
SConnHash *pPrev = pNode->prev, *pNext;
......@@ -242,7 +245,8 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
pNext = pNode->next;
pCache->total--;
pCache->count[hash]--;
//tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode,
// tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port,
// pNode->connType, hash, pNode,
// pCache->count[hash]);
taosMemPoolFree(pCache->connHashMemPool, (char *)pNode);
pNode = pNext;
......@@ -257,7 +261,7 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) {
SConnCache *pCache = (SConnCache *)handle;
int hash = 0;
char *temp = fqdn;
char * temp = fqdn;
while (*temp) {
hash += *temp;
......@@ -288,4 +292,3 @@ static void rpcUnlockCache(int64_t *lockedBy) {
assert(false);
}
}
source/libs/transport/src/rpcMain.c
浏览文件 @ ae2d77e9
......@@ -13,31 +13,36 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "lz4.h"
#include "os.h"
#include "rpcCache.h"
#include "rpcHead.h"
#include "rpcLog.h"
#include "rpcTcp.h"
#include "rpcUdp.h"
#include "taoserror.h"
#include "tglobal.h"
#include "thash.h"
#include "tidpool.h"
#include "tmd5.h"
#include "tmempool.h"
#include "ttimer.h"
#include "tutil.h"
#include "lz4.h"
#include "tref.h"
#include "taoserror.h"
#include "tglobal.h"
#include "tmsg.h"
#include "transportInt.h"
#include "tref.h"
#include "trpc.h"
#include "thash.h"
#include "rpcLog.h"
#include "rpcUdp.h"
#include "rpcCache.h"
#include "rpcTcp.h"
#include "rpcHead.h"
#include "ttimer.h"
#include "tutil.h"
#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest))
#define rpcHeadFromCont(cont) ((SRpcHead *) ((char*)cont - sizeof(SRpcHead)))
#define rpcContFromHead(msg) (msg + sizeof(SRpcHead))
#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead))
#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead))
#define rpcIsReq(type) (type & 1U)
static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT;
int tsRpcMaxUdpSize = 15000; // bytes
int tsProgressTimer = 100;
// not configurable
int tsRpcMaxRetry;
int tsRpcHeadSize;
int tsRpcOverhead;
#ifndef USE_UV
typedef struct {
int sessions; // number of sessions allowed
......@@ -58,24 +63,24 @@ typedef struct {
int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey);
int32_t refCount;
void *parent;
void *idPool; // handle to ID pool
void *tmrCtrl; // handle to timer
SHashObj *hash; // handle returned by hash utility
void *tcphandle;// returned handle from TCP initialization
void *udphandle;// returned handle from UDP initialization
void *pCache; // connection cache
void * parent;
void * idPool; // handle to ID pool
void * tmrCtrl; // handle to timer
SHashObj * hash; // handle returned by hash utility
void * tcphandle; // returned handle from TCP initialization
void * udphandle; // returned handle from UDP initialization
void * pCache; // connection cache
pthread_mutex_t mutex;
struct SRpcConn *connList; // connection list
} SRpcInfo;
typedef struct {
SRpcInfo *pRpc; // associated SRpcInfo
SRpcInfo * pRpc; // associated SRpcInfo
SEpSet epSet; // ip list provided by app
void *ahandle; // handle provided by app
void * ahandle; // handle provided by app
struct SRpcConn *pConn; // pConn allocated
tmsg_t msgType; // message type
uint8_t *pCont; // content provided by app
uint8_t * pCont; // content provided by app
int32_t contLen; // content length
int32_t code; // error code
int16_t numOfTry; // number of try for different servers
......@@ -83,14 +88,21 @@ typedef struct {
int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type
int64_t rid; // refId returned by taosAddRef
SRpcMsg *pRsp; // for synchronous API
tsem_t *pSem; // for synchronous API
SEpSet *pSet; // for synchronous API
SRpcMsg * pRsp; // for synchronous API
tsem_t * pSem; // for synchronous API
SEpSet * pSet; // for synchronous API
char msg[0]; // RpcHead starts from here
} SRpcReqContext;
#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest))
#define rpcHeadFromCont(cont) ((SRpcHead *)((char *)cont - sizeof(SRpcHead)))
#define rpcContFromHead(msg) (msg + sizeof(SRpcHead))
#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead))
#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead))
#define rpcIsReq(type) (type & 1U)
typedef struct SRpcConn {
char info[48];// debug info: label + pConn + ahandle
char info[48]; // debug info: label + pConn + ahandle
int sid; // session ID
uint32_t ownId; // own link ID
uint32_t peerId; // peer link ID
......@@ -110,34 +122,25 @@ typedef struct SRpcConn {
uint16_t inTranId; // transcation ID for incoming msg
tmsg_t outType; // message type for outgoing request
tmsg_t inType; // message type for incoming request
void *chandle; // handle passed by TCP/UDP connection layer
void *ahandle; // handle provided by upper app layter
void * chandle; // handle passed by TCP/UDP connection layer
void * ahandle; // handle provided by upper app layter
int retry; // number of retry for sending request
int tretry; // total retry
void *pTimer; // retry timer to monitor the response
void *pIdleTimer; // idle timer
char *pRspMsg; // response message including header
void * pTimer; // retry timer to monitor the response
void * pIdleTimer; // idle timer
char * pRspMsg; // response message including header
int rspMsgLen; // response messag length
char *pReqMsg; // request message including header
char * pReqMsg; // request message including header
int reqMsgLen; // request message length
SRpcInfo *pRpc; // the associated SRpcInfo
SRpcInfo * pRpc; // the associated SRpcInfo
int8_t connType; // connection type
int64_t lockedBy; // lock for connection
SRpcReqContext *pContext; // request context
} SRpcConn;
static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT;
int tsRpcMaxUdpSize = 15000; // bytes
int tsProgressTimer = 100;
// not configurable
int tsRpcMaxRetry;
int tsRpcHeadSize;
int tsRpcOverhead;
static int tsRpcRefId = -1;
static int32_t tsRpcNum = 0;
//static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
// static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
// server:0 client:1 tcp:2 udp:0
#define RPC_CONN_UDPS 0
......@@ -146,18 +149,10 @@ static int32_t tsRpcNum = 0;
#define RPC_CONN_TCPC 3
void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = {
taosInitUdpConnection,
taosInitUdpConnection,
taosInitTcpServer,
taosInitTcpClient
};
taosInitUdpConnection, taosInitUdpConnection, taosInitTcpServer, taosInitTcpClient};
void (*taosCleanUpConn[])(void *thandle) = {
taosCleanUpUdpConnection,
taosCleanUpUdpConnection,
taosCleanUpTcpServer,
taosCleanUpTcpClient
};
void (*taosCleanUpConn[])(void *thandle) = {taosCleanUpUdpConnection, taosCleanUpUdpConnection, taosCleanUpTcpServer,
taosCleanUpTcpClient};
void (*taosStopConn[])(void *thandle) = {
taosStopUdpConnection,
......@@ -167,11 +162,7 @@ void (*taosStopConn[])(void *thandle) = {
};
int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
taosSendUdpData,
taosSendUdpData,
taosSendTcpData,
taosSendTcpData
};
taosSendUdpData, taosSendUdpData, taosSendTcpData, taosSendTcpData};
void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port) = {
taosOpenUdpConnection,
......@@ -180,12 +171,7 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port
taosOpenTcpClientConnection,
};
void (*taosCloseConn[])(void *chandle) = {
NULL,
NULL,
taosCloseTcpConnection,
taosCloseTcpConnection
};
void (*taosCloseConn[])(void *chandle) = {NULL, NULL, taosCloseTcpConnection, taosCloseTcpConnection};
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType);
static void rpcCloseConn(void *thandle);
......@@ -208,7 +194,7 @@ static void rpcProcessIdleTimer(void *param, void *tmrId);
static void rpcProcessProgressTimer(void *param, void *tmrId);
static void rpcFreeMsg(void *msg);
static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen);
static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen);
static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead);
static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen);
static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen);
......@@ -244,22 +230,23 @@ void rpcCleanup(void) {
void *rpcOpen(const SRpcInit *pInit) {
SRpcInfo *pRpc;
//pthread_once(&tsRpcInit, rpcInit);
// pthread_once(&tsRpcInit, rpcInit);
pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo));
if (pRpc == NULL) return NULL;
if(pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label));
if (pInit->label) tstrncpy(pRpc->label, pInit->label, strlen(pInit->label));
pRpc->connType = pInit->connType;
if (pRpc->connType == TAOS_CONN_CLIENT) {
pRpc->numOfThreads = pInit->numOfThreads;
} else {
pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads;
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
}
pRpc->idleTime = pInit->idleTime;
pRpc->localPort = pInit->localPort;
pRpc->afp = pInit->afp;
pRpc->sessions = pInit->sessions+1;
pRpc->sessions = pInit->sessions + 1;
if (pInit->user) tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user));
if (pInit->secret) memcpy(pRpc->secret, pInit->secret, sizeof(pRpc->secret));
if (pInit->ckey) tstrncpy(pRpc->ckey, pInit->ckey, sizeof(pRpc->ckey));
......@@ -279,14 +266,14 @@ void *rpcOpen(const SRpcInit *pInit) {
return NULL;
}
pRpc->idPool = taosInitIdPool(pRpc->sessions-1);
pRpc->idPool = taosInitIdPool(pRpc->sessions - 1);
if (pRpc->idPool == NULL) {
tError("%s failed to init ID pool", pRpc->label);
rpcClose(pRpc);
return NULL;
}
pRpc->tmrCtrl = taosTmrInit(pRpc->sessions*2 + 1, 50, 10000, pRpc->label);
pRpc->tmrCtrl = taosTmrInit(pRpc->sessions * 2 + 1, 50, 10000, pRpc->label);
if (pRpc->tmrCtrl == NULL) {
tError("%s failed to init timers", pRpc->label);
rpcClose(pRpc);
......@@ -302,7 +289,7 @@ void *rpcOpen(const SRpcInit *pInit) {
}
} else {
pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime * 20);
if ( pRpc->pCache == NULL ) {
if (pRpc->pCache == NULL) {
tError("%s failed to init connection cache", pRpc->label);
rpcClose(pRpc);
return NULL;
......@@ -311,10 +298,10 @@ void *rpcOpen(const SRpcInit *pInit) {
pthread_mutex_init(&pRpc->mutex, NULL);
pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label,
pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label,
pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
pRpc->tcphandle = (*taosInitConn[pRpc->connType | RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads,
rpcProcessMsgFromPeer, pRpc);
pRpc->udphandle =
(*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) {
tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort);
......@@ -375,7 +362,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
if (ptr == NULL) return rpcMallocCont(contLen);
char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead);
if (contLen == 0 ) {
if (contLen == 0) {
free(start);
return NULL;
}
......@@ -391,11 +378,11 @@ void *rpcReallocCont(void *ptr, int contLen) {
}
void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {
SRpcInfo *pRpc = (SRpcInfo *)shandle;
SRpcInfo * pRpc = (SRpcInfo *)shandle;
SRpcReqContext *pContext;
int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen);
pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext));
pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext));
pContext->ahandle = pMsg->ahandle;
pContext->pRpc = (SRpcInfo *)shandle;
pContext->epSet = *pEpSet;
......@@ -405,15 +392,14 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t
pContext->oldInUse = pEpSet->inUse;
pContext->connType = RPC_CONN_UDPC;
if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp ) pContext->connType = RPC_CONN_TCPC;
if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp) pContext->connType = RPC_CONN_TCPC;
// connection type is application specific.
// for TDengine, all the query, show commands shall have TCP connection
tmsg_t type = pMsg->msgType;
if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE
|| type == TDMT_VND_FETCH || type == TDMT_MND_VGROUP_LIST
|| type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META
|| type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE)
if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE || type == TDMT_VND_FETCH ||
type == TDMT_MND_VGROUP_LIST || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META ||
type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE)
pContext->connType = RPC_CONN_TCPC;
pContext->rid = taosAddRef(tsRpcRefId, pContext);
......@@ -428,23 +414,23 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
int msgLen = 0;
SRpcConn *pConn = (SRpcConn *)pRsp->handle;
SRpcMsg rpcMsg = *pRsp;
SRpcMsg *pMsg = &rpcMsg;
SRpcMsg * pMsg = &rpcMsg;
SRpcInfo *pRpc = pConn->pRpc;
if ( pMsg->pCont == NULL ) {
if (pMsg->pCont == NULL) {
pMsg->pCont = rpcMallocCont(0);
pMsg->contLen = 0;
}
SRpcHead *pHead = rpcHeadFromCont(pMsg->pCont);
char *msg = (char *)pHead;
char * msg = (char *)pHead;
pMsg->contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen);
msgLen = rpcMsgLenFromCont(pMsg->contLen);
rpcLockConn(pConn);
if ( pConn->inType == 0 || pConn->user[0] == 0 ) {
if (pConn->inType == 0 || pConn->user[0] == 0) {
tError("%s, connection is already released, rsp wont be sent", pConn->info);
rpcUnlockConn(pConn);
rpcFreeCont(pMsg->pCont);
......@@ -454,7 +440,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
// set msg header
pHead->version = 1;
pHead->msgType = pConn->inType+1;
pHead->msgType = pConn->inType + 1;
pHead->spi = pConn->spi;
pHead->encrypt = pConn->encrypt;
pHead->tranId = pConn->inTranId;
......@@ -463,7 +449,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
pHead->linkUid = pConn->linkUid;
pHead->port = htons(pConn->localPort);
pHead->code = htonl(pMsg->code);
pHead->ahandle = (uint64_t) pConn->ahandle;
pHead->ahandle = (uint64_t)pConn->ahandle;
// set pConn parameters
pConn->inType = 0;
......@@ -482,8 +468,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
rpcSendMsgToPeer(pConn, msg, msgLen);
// if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured
if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY)
pConn->secured = 1; // connection shall be secured
if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) pConn->secured = 1; // connection shall be secured
if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg);
pConn->pReqMsg = NULL;
......@@ -527,7 +512,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
SRpcReqContext *pContext;
pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext));
pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext));
memset(pRsp, 0, sizeof(SRpcMsg));
......@@ -567,7 +552,6 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
}
void rpcCancelRequest(int64_t rid) {
SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid);
if (pContext == NULL) return;
......@@ -577,7 +561,7 @@ void rpcCancelRequest(int64_t rid) {
}
static void rpcFreeMsg(void *msg) {
if ( msg ) {
if (msg) {
char *temp = (char *)msg - sizeof(SRpcReqContext);
free(temp);
tTrace("free mem: %p", temp);
......@@ -604,7 +588,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort,
pConn->connType = connType;
if (taosOpenConn[connType]) {
void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle;
void *shandle = (connType & RPC_CONN_TCP) ? pRpc->tcphandle : pRpc->udphandle;
pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort);
if (pConn->chandle == NULL) {
tError("failed to connect to:%s:%d", taosIpStr(pConn->peerIp), pConn->peerPort);
......@@ -629,9 +613,10 @@ static void rpcReleaseConn(SRpcConn *pConn) {
taosTmrStopA(&pConn->pTimer);
taosTmrStopA(&pConn->pIdleTimer);
if ( pRpc->connType == TAOS_CONN_SERVER) {
if (pRpc->connType == TAOS_CONN_SERVER) {
char hashstr[40] = {0};
size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType);
size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId,
pConn->connType);
taosHashRemove(pRpc->hash, hashstr, size);
rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg
pConn->pRspMsg = NULL;
......@@ -682,8 +667,7 @@ static void rpcCloseConn(void *thandle) {
rpcLockConn(pConn);
if (pConn->user[0])
rpcReleaseConn(pConn);
if (pConn->user[0]) rpcReleaseConn(pConn);
rpcUnlockConn(pConn);
}
......@@ -717,7 +701,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
char hashstr[40] = {0};
SRpcHead *pHead = (SRpcHead *)pRecv->msg;
size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType);
size_t size =
snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType);
// check if it is already allocated
SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size));
......@@ -767,7 +752,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
}
taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, hashstr);
tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid,
hashstr);
}
return pConn;
......@@ -805,10 +791,11 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) {
static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
SRpcConn *pConn;
SRpcInfo *pRpc = pContext->pRpc;
SEpSet *pEpSet = &pContext->epSet;
SEpSet * pEpSet = &pContext->epSet;
pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType);
if ( pConn == NULL || pConn->user[0] == 0) {
pConn =
rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType);
if (pConn == NULL || pConn->user[0] == 0) {
pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType);
}
......@@ -825,13 +812,11 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
}
static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) {
if (pConn->peerId == 0) {
pConn->peerId = pHead->sourceId;
} else {
if (pConn->peerId != pHead->sourceId) {
tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info,
pConn->peerId, pHead->sourceId);
tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, pConn->peerId, pHead->sourceId);
return TSDB_CODE_RPC_INVALID_VALUE;
}
}
......@@ -856,8 +841,7 @@ static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) {
}
if (pConn->inType != 0) {
tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info,
pConn->inTranId, pHead->tranId);
tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, pConn->inTranId, pHead->tranId);
return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED;
}
......@@ -961,18 +945,22 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
if (TMSG_INDEX(pHead->msgType) >= TDMT_MAX || TMSG_INDEX(pHead->msgType) <= 0) {
tDebug("%s sid:%d, invalid message type:%d", pRpc->label, sid, pHead->msgType);
terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; return NULL;
terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE;
return NULL;
}
if (sid < 0 || sid >= pRpc->sessions) {
tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid,
pRpc->sessions, TMSG_INFO(pHead->msgType));
terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL;
tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, pRpc->sessions,
TMSG_INFO(pHead->msgType));
terrno = TSDB_CODE_RPC_INVALID_SESSION_ID;
return NULL;
}
if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) {
tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, TMSG_INFO(pHead->msgType));
terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL;
tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion,
TMSG_INFO(pHead->msgType));
terrno = TSDB_CODE_RPC_INVALID_VERSION;
return NULL;
}
pConn = rpcGetConnObj(pRpc, sid, pRecv);
......@@ -1004,7 +992,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
// decrypt here
}
if ( rpcIsReq(pHead->msgType) ) {
if (rpcIsReq(pHead->msgType)) {
pConn->connType = pRecv->connType;
terrno = rpcProcessReqHead(pConn, pHead);
......@@ -1013,7 +1001,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
// client shall send the request within tsRpcTime again for UDP, double it
if (pConn->connType != RPC_CONN_TCPS)
pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer*2, pConn, pRpc->tmrCtrl);
pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer * 2, pConn, pRpc->tmrCtrl);
} else {
terrno = rpcProcessRspHead(pConn, pHead);
*ppContext = pConn->pContext;
......@@ -1026,7 +1014,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
}
static void doRpcReportBrokenLinkToServer(void *param, void *id) {
SRpcMsg *pRpcMsg = (SRpcMsg *)(param);
SRpcMsg * pRpcMsg = (SRpcMsg *)(param);
SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle);
SRpcInfo *pRpc = pConn->pRpc;
(*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL);
......@@ -1100,13 +1088,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
taosIpPort2String(pRecv->ip, pRecv->port, ipstr);
if (TMSG_INDEX(pHead->msgType) >= 1 && TMSG_INDEX(pHead->msgType) < TDMT_MAX) {
tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label,
pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen,
pHead->sourceId, pHead->destId, pHead->tranId, pHead->code);
tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn,
(void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen, pHead->sourceId,
pHead->destId, pHead->tranId, pHead->code);
} else {
tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label,
pConn, (void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen,
pHead->sourceId, pHead->destId, pHead->tranId, pHead->code);
tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn,
(void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen, pHead->sourceId, pHead->destId,
pHead->tranId, pHead->code);
}
int32_t code = terrno;
......@@ -1118,9 +1106,11 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
rpcCloseConn(pConn);
}
if (TMSG_INDEX(pHead->msgType) + 1 > 1 && TMSG_INDEX(pHead->msgType) + 1 < TDMT_MAX) {
tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType+1), code);
tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle,
TMSG_INFO(pHead->msgType + 1), code);
} else {
tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), code);
tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle,
TMSG_INFO(pHead->msgType), code);
}
}
} else { // msg is passed to app only parsing is ok
......@@ -1144,8 +1134,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
} else {
// for asynchronous API
SEpSet *pEpSet = NULL;
if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect)
pEpSet = &pContext->epSet;
if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) pEpSet = &pContext->epSet;
(*pRpc->cfp)(pRpc->parent, pMsg, pEpSet);
}
......@@ -1155,7 +1144,6 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
}
static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) {
SRpcInfo *pRpc = pConn->pRpc;
SRpcMsg rpcMsg;
......@@ -1180,14 +1168,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
// for UDP, port may be changed by server, the port in epSet shall be used for cache
if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) {
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], pConn->connType);
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse],
pConn->connType);
} else {
rpcCloseConn(pConn);
}
if (pHead->code == TSDB_CODE_RPC_REDIRECT) {
pContext->numOfTry = 0;
SEpSet *pEpSet = (SEpSet*)pHead->content;
SEpSet *pEpSet = (SEpSet *)pHead->content;
if (pEpSet->numOfEps > 0) {
memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet));
tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps,
......@@ -1200,7 +1189,8 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
}
rpcSendReqToServer(pRpc, pContext);
rpcFreeCont(rpcMsg.pCont);
} else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || pHead->code == TSDB_CODE_DND_OFFLINE) {
} else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY ||
pHead->code == TSDB_CODE_DND_OFFLINE) {
pContext->code = pHead->code;
rpcProcessConnError(pContext, NULL);
rpcFreeCont(rpcMsg.pCont);
......@@ -1218,7 +1208,7 @@ static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) {
memset(msg, 0, sizeof(SRpcHead));
pHead = (SRpcHead *)msg;
pHead->version = 1;
pHead->msgType = pConn->inType+1;
pHead->msgType = pConn->inType + 1;
pHead->spi = pConn->spi;
pHead->encrypt = 0;
pHead->tranId = pConn->inTranId;
......@@ -1258,7 +1248,7 @@ static void rpcSendReqHead(SRpcConn *pConn) {
static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) {
SRpcHead *pRecvHead, *pReplyHead;
char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t) ];
char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t)];
uint32_t timeStamp;
int msgLen;
......@@ -1295,7 +1285,7 @@ static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) {
static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
SRpcHead *pHead = rpcHeadFromCont(pContext->pCont);
char *msg = (char *)pHead;
char * msg = (char *)pHead;
int msgLen = rpcMsgLenFromCont(pContext->contLen);
tmsg_t msgType = pContext->msgType;
......@@ -1317,7 +1307,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
pHead->msgType = msgType;
pHead->encrypt = 0;
pConn->tranId++;
if ( pConn->tranId == 0 ) pConn->tranId++;
if (pConn->tranId == 0) pConn->tranId++;
pHead->tranId = pConn->tranId;
pHead->sourceId = pConn->ownId;
pHead->destId = pConn->peerId;
......@@ -1346,18 +1336,16 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
msgLen = rpcAddAuthPart(pConn, msg, msgLen);
if ( rpcIsReq(pHead->msgType)) {
tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
pConn->info, TMSG_INFO(pHead->msgType), pConn->peerFqdn, pConn->peerPort,
msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
if (rpcIsReq(pHead->msgType)) {
tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType),
pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
} else {
if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
pConn->info, TMSG_INFO(pHead->msgType), pConn->peerIp, pConn->peerPort,
htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType),
pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
}
//tTrace("connection type is: %d", pConn->connType);
// tTrace("connection type is: %d", pConn->connType);
writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle);
if (writtenLen != msgLen) {
......@@ -1369,7 +1357,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
static void rpcProcessConnError(void *param, void *id) {
SRpcReqContext *pContext = (SRpcReqContext *)param;
SRpcInfo *pRpc = pContext->pRpc;
SRpcInfo * pRpc = pContext->pRpc;
SRpcMsg rpcMsg;
if (pRpc == NULL) {
......@@ -1379,7 +1367,7 @@ static void rpcProcessConnError(void *param, void *id) {
tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle);
if (pContext->numOfTry >= pContext->epSet.numOfEps || pContext->msgType == TDMT_VND_FETCH) {
rpcMsg.msgType = pContext->msgType+1;
rpcMsg.msgType = pContext->msgType + 1;
rpcMsg.ahandle = pContext->ahandle;
rpcMsg.code = pContext->code;
rpcMsg.pCont = NULL;
......@@ -1411,7 +1399,8 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl);
} else {
// close the connection
tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort);
tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn,
pConn->peerPort);
if (pConn->pContext) {
pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
pConn->pContext->pConn = NULL;
......@@ -1460,7 +1449,7 @@ static void rpcProcessProgressTimer(void *param, void *tmrId) {
rpcUnlockConn(pConn);
}
static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) {
static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) {
SRpcHead *pHead = rpcHeadFromCont(pCont);
int32_t finalLen = 0;
int overhead = sizeof(SRpcComp);
......@@ -1469,7 +1458,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) {
return contLen;
}
char *buf = malloc (contLen + overhead + 8); // 8 extra bytes
char *buf = malloc(contLen + overhead + 8); // 8 extra bytes
if (buf == NULL) {
tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen);
return contLen;
......@@ -1502,7 +1491,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) {
static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) {
int overhead = sizeof(SRpcComp);
SRpcHead *pNewHead = NULL;
uint8_t *pCont = pHead->content;
uint8_t * pCont = pHead->content;
SRpcComp *pComp = (SRpcComp *)pHead->content;
if (pHead->comp) {
......@@ -1516,7 +1505,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) {
if (pNewHead) {
int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead;
int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char *)pNewHead->content, compLen, contLen);
int origLen = LZ4_decompress_safe((char *)(pCont + overhead), (char *)pNewHead->content, compLen, contLen);
assert(origLen == contLen);
memcpy(pNewHead, pHead, sizeof(SRpcHead));
......@@ -1582,19 +1571,19 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) {
SRpcHead *pHead = (SRpcHead *)msg;
int code = 0;
if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)){
if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) {
// secured link, or no authentication
pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen);
// tTrace("%s, secured link, no auth is required", pConn->info);
return 0;
}
if ( !rpcIsReq(pHead->msgType) ) {
if (!rpcIsReq(pHead->msgType)) {
// for response, if code is auth failure, it shall bypass the auth process
code = htonl(pHead->code);
if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE ||
code == TSDB_CODE_RPC_INVALID_VERSION ||
code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) {
code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED ||
code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) {
pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen);
// tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code);
return 0;
......@@ -1613,12 +1602,12 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) {
tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta);
code = TSDB_CODE_RPC_INVALID_TIME_STAMP;
} else {
if (rpcAuthenticateMsg(pHead, msgLen-TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) {
if (rpcAuthenticateMsg(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) {
tDebug("%s, authentication failed, msg discarded", pConn->info);
code = TSDB_CODE_RPC_AUTH_FAILURE;
} else {
pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest);
if ( !rpcIsReq(pHead->msgType) ) pConn->secured = 1; // link is secured for client
if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client
// tTrace("%s, message is authenticated", pConn->info);
}
}
......@@ -1647,13 +1636,9 @@ static void rpcUnlockConn(SRpcConn *pConn) {
}
}
static void rpcAddRef(SRpcInfo *pRpc)
{
atomic_add_fetch_32(&pRpc->refCount, 1);
}
static void rpcAddRef(SRpcInfo *pRpc) { atomic_add_fetch_32(&pRpc->refCount, 1); }
static void rpcDecRef(SRpcInfo *pRpc)
{
static void rpcDecRef(SRpcInfo *pRpc) {
if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) {
rpcCloseConnCache(pRpc->pCache);
taosHashCleanup(pRpc->hash);
......@@ -1668,4 +1653,4 @@ static void rpcDecRef(SRpcInfo *pRpc)
atomic_sub_fetch_32(&tsRpcNum, 1);
}
}
#endif
source/libs/transport/src/rpcTcp.c
浏览文件 @ ae2d77e9
......@@ -13,24 +13,24 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "rpcTcp.h"
#include "os.h"
#include "tutil.h"
#include "rpcHead.h"
#include "rpcLog.h"
#include "taosdef.h"
#include "taoserror.h"
#include "rpcLog.h"
#include "rpcHead.h"
#include "rpcTcp.h"
#include "tutil.h"
typedef struct SFdObj {
void *signature;
void * signature;
SOCKET fd; // TCP socket FD
void *thandle; // handle from upper layer, like TAOS
void * thandle; // handle from upper layer, like TAOS
uint32_t ip;
uint16_t port;
int16_t closedByApp; // 1: already closed by App
struct SThreadObj *pThreadObj;
struct SFdObj *prev;
struct SFdObj *next;
struct SFdObj * prev;
struct SFdObj * next;
} SFdObj;
typedef struct SThreadObj {
......@@ -43,7 +43,7 @@ typedef struct SThreadObj {
int numOfFds;
int threadId;
char label[TSDB_LABEL_LEN];
void *shandle; // handle passed by upper layer during server initialization
void * shandle; // handle passed by upper layer during server initialization
void *(*processData)(SRecvInfo *pPacket);
} SThreadObj;
......@@ -67,11 +67,11 @@ typedef struct {
pthread_t thread;
} SServerObj;
static void *taosProcessTcpData(void *param);
static void * taosProcessTcpData(void *param);
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd);
static void taosFreeFdObj(SFdObj *pFdObj);
static void taosReportBrokenLink(SFdObj *pFdObj);
static void *taosAcceptTcpConnection(void *arg);
static void * taosAcceptTcpConnection(void *arg);
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
SServerObj *pServerObj;
......@@ -110,7 +110,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
if (pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
terrno = TAOS_SYSTEM_ERROR(errno);
for (int j=0; j<i; ++j) free(pServerObj->pThreadObj[j]);
for (int j = 0; j < i; ++j) free(pServerObj->pThreadObj[j]);
free(pServerObj->pThreadObj);
free(pServerObj);
return NULL;
......@@ -172,8 +172,10 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
return (void *)pServerObj;
}
static void taosStopTcpThread(SThreadObj* pThreadObj) {
if (pThreadObj == NULL) { return;}
static void taosStopTcpThread(SThreadObj *pThreadObj) {
if (pThreadObj == NULL) {
return;
}
// save thread into local variable and signal thread to stop
pthread_t thread = pThreadObj->thread;
if (!taosCheckPthreadValid(thread)) {
......@@ -227,8 +229,8 @@ static void *taosAcceptTcpConnection(void *arg) {
SOCKET connFd = -1;
struct sockaddr_in caddr;
int threadId = 0;
SThreadObj *pThreadObj;
SServerObj *pServerObj;
SThreadObj * pThreadObj;
SServerObj * pServerObj;
pServerObj = (SServerObj *)arg;
tDebug("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
......@@ -253,7 +255,7 @@ static void *taosAcceptTcpConnection(void *arg) {
}
taosKeepTcpAlive(connFd);
struct timeval to={5, 0};
struct timeval to = {5, 0};
int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
if (ret != 0) {
taosCloseSocket(connFd);
......@@ -262,7 +264,6 @@ static void *taosAcceptTcpConnection(void *arg) {
continue;
}
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj[threadId];
......@@ -297,7 +298,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
tstrncpy(pClientObj->label, label, sizeof(pClientObj->label));
pClientObj->numOfThreads = numOfThreads;
pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*));
pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj *));
if (pClientObj->pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
tfree(pClientObj);
......@@ -314,7 +315,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
if (pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
terrno = TAOS_SYSTEM_ERROR(errno);
for (int j=0; j<i; ++j) free(pClientObj->pThreadObj[j]);
for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]);
free(pClientObj);
pthread_attr_destroy(&thattr);
return NULL;
......@@ -364,14 +365,14 @@ void taosStopTcpClient(void *chandle) {
if (pClientObj == NULL) return;
tDebug ("%s TCP client is stopped", pClientObj->label);
tDebug("%s TCP client is stopped", pClientObj->label);
}
void taosCleanUpTcpClient(void *chandle) {
SClientObj *pClientObj = chandle;
if (pClientObj == NULL) return;
for (int i = 0; i < pClientObj->numOfThreads; ++i) {
SThreadObj *pThreadObj= pClientObj->pThreadObj[i];
SThreadObj *pThreadObj = pClientObj->pThreadObj[i];
taosStopTcpThread(pThreadObj);
}
......@@ -381,7 +382,7 @@ void taosCleanUpTcpClient(void *chandle) {
}
void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
SClientObj * pClientObj = shandle;
SClientObj *pClientObj = shandle;
int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads;
atomic_store_32(&pClientObj->index, index + 1);
SThreadObj *pThreadObj = pClientObj->pThreadObj[index];
......@@ -396,8 +397,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
struct sockaddr_in sin;
uint16_t localPort = 0;
unsigned int addrlen = sizeof(sin);
if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 &&
sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
localPort = (uint16_t)ntohs(sin.sin_port);
}
......@@ -407,8 +407,11 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
pFdObj->thandle = thandle;
pFdObj->port = port;
pFdObj->ip = ip;
tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
char ipport[40] = {0};
taosIpPort2String(ip, port, ipport);
tDebug("%s %p TCP connection to %s is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle,
ipport, localPort, pFdObj, pThreadObj->numOfFds);
} else {
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
taosCloseSocket(fd);
......@@ -441,7 +444,6 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand
}
static void taosReportBrokenLink(SFdObj *pFdObj) {
SThreadObj *pThreadObj = pFdObj->pThreadObj;
// notify the upper layer, so it will clean the associated context
......@@ -466,7 +468,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
SRpcHead rpcHead;
int32_t msgLen, leftLen, retLen, headLen;
char *buffer, *msg;
char * buffer, *msg;
SThreadObj *pThreadObj = pFdObj->pThreadObj;
......@@ -483,7 +485,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
return -1;
} else {
tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer);
tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd,
buffer);
}
msg = buffer + tsRpcOverhead;
......@@ -491,8 +494,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s %p read error, leftLen:%d retLen:%d FD:%p",
pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj);
tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj);
free(buffer);
return -1;
}
......@@ -519,8 +521,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
#define maxEvents 10
static void *taosProcessTcpData(void *param) {
SThreadObj *pThreadObj = param;
SFdObj *pFdObj;
SThreadObj * pThreadObj = param;
SFdObj * pFdObj;
struct epoll_event events[maxEvents];
SRecvInfo recvInfo;
......@@ -569,7 +571,7 @@ static void *taosProcessTcpData(void *param) {
if (pThreadObj->stop) break;
}
if (pThreadObj->pollFd >=0) {
if (pThreadObj->pollFd >= 0) {
EpollClose(pThreadObj->pollFd);
pThreadObj->pollFd = -1;
}
......@@ -620,7 +622,6 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
}
static void taosFreeFdObj(SFdObj *pFdObj) {
if (pFdObj == NULL) return;
if (pFdObj->signature != pFdObj) return;
......@@ -638,8 +639,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pThreadObj->numOfFds--;
if (pThreadObj->numOfFds < 0)
tError("%s %p TCP thread:%d, number of FDs is negative!!!",
pThreadObj->label, pFdObj->thandle, pThreadObj->threadId);
tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle,
pThreadObj->threadId);
if (pFdObj->prev) {
(pFdObj->prev)->next = pFdObj->next;
......@@ -653,8 +654,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pthread_mutex_unlock(&pThreadObj->mutex);
tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds);
tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj,
pFdObj->fd, pThreadObj->numOfFds);
tfree(pFdObj);
}
source/libs/transport/src/rpcUdp.c
浏览文件 @ ae2d77e9
......@@ -13,14 +13,14 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "rpcUdp.h"
#include "os.h"
#include "ttimer.h"
#include "tutil.h"
#include "rpcHead.h"
#include "rpcLog.h"
#include "taosdef.h"
#include "taoserror.h"
#include "rpcLog.h"
#include "rpcUdp.h"
#include "rpcHead.h"
#include "ttimer.h"
#include "tutil.h"
#define RPC_MAX_UDP_CONNS 256
#define RPC_MAX_UDP_PKTS 1000
......@@ -34,9 +34,9 @@ typedef struct {
uint16_t localPort; // local port
char label[TSDB_LABEL_LEN]; // copy from udpConnSet;
pthread_t thread;
void *hash;
void *shandle; // handle passed by upper layer during server initialization
void *pSet;
void * hash;
void * shandle; // handle passed by upper layer during server initialization
void * pSet;
void *(*processData)(SRecvInfo *pRecv);
char *buffer; // buffer to receive data
} SUdpConn;
......@@ -46,7 +46,7 @@ typedef struct {
int server;
uint32_t ip; // local IP
uint16_t port; // local Port
void *shandle; // handle passed by upper layer during server initialization
void * shandle; // handle passed by upper layer during server initialization
int threads;
char label[TSDB_LABEL_LEN];
void *(*fp)(SRecvInfo *pPacket);
......@@ -56,7 +56,7 @@ typedef struct {
static void *taosRecvUdpData(void *param);
void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) {
SUdpConn *pConn;
SUdpConn * pConn;
SUdpConnSet *pSet;
int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn);
......@@ -98,8 +98,8 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
struct sockaddr_in sin;
unsigned int addrlen = sizeof(sin);
if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 &&
sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET &&
addrlen == sizeof(sin)) {
pConn->localPort = (uint16_t)ntohs(sin.sin_port);
}
......@@ -130,14 +130,14 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
void taosStopUdpConnection(void *handle) {
SUdpConnSet *pSet = (SUdpConnSet *)handle;
SUdpConn *pConn;
SUdpConn * pConn;
if (pSet == NULL) return;
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR);
if (pConn->fd >=0) taosCloseSocket(pConn->fd);
if (pConn->fd >= 0) shutdown(pConn->fd, SHUT_RDWR);
if (pConn->fd >= 0) taosCloseSocket(pConn->fd);
pConn->fd = -1;
}
......@@ -155,13 +155,13 @@ void taosStopUdpConnection(void *handle) {
void taosCleanUpUdpConnection(void *handle) {
SUdpConnSet *pSet = (SUdpConnSet *)handle;
SUdpConn *pConn;
SUdpConn * pConn;
if (pSet == NULL) return;
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
if (pConn->fd >=0) taosCloseSocket(pConn->fd);
if (pConn->fd >= 0) taosCloseSocket(pConn->fd);
}
tDebug("%s UDP is cleaned up", pSet->label);
......@@ -182,7 +182,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
}
static void *taosRecvUdpData(void *param) {
SUdpConn *pConn = param;
SUdpConn * pConn = param;
struct sockaddr_in sourceAdd;
ssize_t dataLen;
unsigned int addLen;
......@@ -218,7 +218,7 @@ static void *taosRecvUdpData(void *param) {
}
int32_t size = dataLen + tsRpcOverhead;
char *tmsg = malloc(size);
char * tmsg = malloc(size);
if (NULL == tmsg) {
tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen);
continue;
......@@ -257,4 +257,3 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c
return ret;
}
source/libs/transport/src/transport.c
浏览文件 @ ae2d77e9
......@@ -12,3 +12,735 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef USE_UV
#include <uv.h>
#include "lz4.h"
#include "os.h"
#include "rpcCache.h"
#include "rpcHead.h"
#include "rpcLog.h"
#include "rpcTcp.h"
#include "rpcUdp.h"
#include "taoserror.h"
#include "tglobal.h"
#include "thash.h"
#include "tidpool.h"
#include "tmd5.h"
#include "tmempool.h"
#include "tmsg.h"
#include "transportInt.h"
#include "tref.h"
#include "trpc.h"
#include "ttimer.h"
#include "tutil.h"
#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member)))
#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext))
static const char* notify = "a";
typedef struct {
int sessions; // number of sessions allowed
int numOfThreads; // number of threads to process incoming messages
int idleTime; // milliseconds;
uint16_t localPort;
int8_t connType;
int index; // for UDP server only, round robin for multiple threads
char label[TSDB_LABEL_LEN];
char user[TSDB_UNI_LEN]; // meter ID
char spi; // security parameter index
char encrypt; // encrypt algorithm
char secret[TSDB_PASSWORD_LEN]; // secret for the link
char ckey[TSDB_PASSWORD_LEN]; // ciphering key
void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey);
int32_t refCount;
void* parent;
void* idPool; // handle to ID pool
void* tmrCtrl; // handle to timer
SHashObj* hash; // handle returned by hash utility
void* tcphandle; // returned handle from TCP initialization
void* udphandle; // returned handle from UDP initialization
void* pCache; // connection cache
pthread_mutex_t mutex;
struct SRpcConn* connList; // connection list
} SRpcInfo;
typedef struct {
SRpcInfo* pRpc; // associated SRpcInfo
SEpSet epSet; // ip list provided by app
void* ahandle; // handle provided by app
struct SRpcConn* pConn; // pConn allocated
tmsg_t msgType; // message type
uint8_t* pCont; // content provided by app
int32_t contLen; // content length
int32_t code; // error code
int16_t numOfTry; // number of try for different servers
int8_t oldInUse; // server EP inUse passed by app
int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type
int64_t rid; // refId returned by taosAddRef
SRpcMsg* pRsp; // for synchronous API
tsem_t* pSem; // for synchronous API
SEpSet* pSet; // for synchronous API
char msg[0]; // RpcHead starts from here
} SRpcReqContext;
typedef struct SThreadObj {
pthread_t thread;
uv_pipe_t* pipe;
int fd;
uv_loop_t* loop;
uv_async_t* workerAsync; //
queue conn;
pthread_mutex_t connMtx;
void* shandle;
} SThreadObj;
#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest))
#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead)))
#define rpcContFromHead(msg) (msg + sizeof(SRpcHead))
#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead))
#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead))
#define rpcIsReq(type) (type & 1U)
typedef struct SServerObj {
pthread_t thread;
uv_tcp_t server;
uv_loop_t* loop;
int workerIdx;
int numOfThread;
SThreadObj** pThreadObj;
uv_pipe_t** pipe;
uint32_t ip;
uint32_t port;
} SServerObj;
typedef struct SConnBuffer {
char* buf;
int len;
int cap;
int left;
} SConnBuffer;
typedef struct SRpcConn {
uv_tcp_t* pTcp;
uv_write_t* pWriter;
uv_timer_t* pTimer;
uv_async_t* pWorkerAsync;
queue queue;
int ref;
int persist; // persist connection or not
SConnBuffer connBuf; // read buf,
SConnBuffer writeBuf; // write buf
int count;
void* shandle; // rpc init
void* ahandle; //
void* hostThread;
// del later
char secured;
int spi;
char info[64];
char user[TSDB_UNI_LEN]; // user ID for the link
char secret[TSDB_PASSWORD_LEN];
char ckey[TSDB_PASSWORD_LEN]; // ciphering key
} SRpcConn;
// auth function
static int uvAuthMsg(SRpcConn* pConn, char* msg, int msgLen);
static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey);
static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey);
static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen);
// compress data
static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen);
static SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead);
static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
static void uvOnTimeoutCb(uv_timer_t* handle);
static void uvOnWriteCb(uv_write_t* req, int status);
static void uvOnAcceptCb(uv_stream_t* stream, int status);
static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf);
static void uvWorkerAsyncCb(uv_async_t* handle);
static SRpcConn* connCreate();
static void connDestroy(SRpcConn* conn);
static void uvConnDestroy(uv_handle_t* handle);
static void* workerThread(void* arg);
static void* acceptThread(void* arg);
void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) {
SServerObj* srv = calloc(1, sizeof(SServerObj));
srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
srv->numOfThread = numOfThreads;
srv->workerIdx = 0;
srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*));
srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*));
srv->ip = ip;
srv->port = port;
uv_loop_init(srv->loop);
for (int i = 0; i < srv->numOfThread; i++) {
SThreadObj* thrd = (SThreadObj*)calloc(1, sizeof(SThreadObj));
srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t));
int fds[2];
if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) {
return NULL;
}
uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1);
uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write
thrd->shandle = shandle;
thrd->fd = fds[0];
thrd->pipe = &(srv->pipe[i][1]); // init read
int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd));
if (err == 0) {
tDebug("sucess to create worker-thread %d", i);
// printf("thread %d create\n", i);
} else {
// TODO: clear all other resource later
tError("failed to create worker-thread %d", i);
}
srv->pThreadObj[i] = thrd;
}
int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv);
if (err == 0) {
tDebug("success to create accept-thread");
} else {
// clear all resource later
}
return srv;
}
void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
/*
* formate of data buffer:
* |<-------SRpcReqContext------->|<------------data read from socket----------->|
*/
static const int CAPACITY = 1024;
SRpcConn* ctx = handle->data;
SConnBuffer* pBuf = &ctx->connBuf;
if (pBuf->cap == 0) {
pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char));
pBuf->len = 0;
pBuf->cap = CAPACITY;
pBuf->left = -1;
buf->base = pBuf->buf + RPC_RESERVE_SIZE;
buf->len = CAPACITY;
} else {
if (pBuf->len >= pBuf->cap) {
if (pBuf->left == -1) {
pBuf->cap *= 2;
pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE);
} else if (pBuf->len + pBuf->left > pBuf->cap) {
pBuf->cap = pBuf->len + pBuf->left;
pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE);
}
}
buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE;
buf->len = pBuf->cap - pBuf->len;
}
}
// check data read from socket completely or not
//
static bool isReadAll(SConnBuffer* data) {
// TODO(yihao): handle pipeline later
SRpcHead rpcHead;
int32_t headLen = sizeof(rpcHead);
if (data->len >= headLen) {
memcpy((char*)&rpcHead, data->buf + RPC_RESERVE_SIZE, headLen);
int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
if (msgLen > data->len) {
data->left = msgLen - data->len;
return false;
} else {
return true;
}
} else {
return false;
}
}
static void uvDoProcess(SRecvInfo* pRecv) {
SRpcHead* pHead = (SRpcHead*)pRecv->msg;
SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle;
SRpcConn* pConn = pRecv->thandle;
tDump(pRecv->msg, pRecv->msgLen);
terrno = 0;
SRpcReqContext* pContest;
// do auth and check
}
static int uvAuthMsg(SRpcConn* pConn, char* msg, int len) {
SRpcHead* pHead = (SRpcHead*)msg;
int code = 0;
if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) {
// secured link, or no authentication
pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen);
// tTrace("%s, secured link, no auth is required", pConn->info);
return 0;
}
if (!rpcIsReq(pHead->msgType)) {
// for response, if code is auth failure, it shall bypass the auth process
code = htonl(pHead->code);
if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE ||
code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED ||
code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) {
pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen);
// tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code);
return 0;
}
}
code = 0;
if (pHead->spi == pConn->spi) {
// authentication
SRpcDigest* pDigest = (SRpcDigest*)((char*)pHead + len - sizeof(SRpcDigest));
int32_t delta;
delta = (int32_t)htonl(pDigest->timeStamp);
delta -= (int32_t)taosGetTimestampSec();
if (abs(delta) > 900) {
tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta);
code = TSDB_CODE_RPC_INVALID_TIME_STAMP;
} else {
if (rpcAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) {
// tDebug("%s, authentication failed, msg discarded", pConn->info);
code = TSDB_CODE_RPC_AUTH_FAILURE;
} else {
pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest);
if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client
// tTrace("%s, message is authenticated", pConn->info);
}
}
} else {
tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi);
code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED;
}
return code;
}
// refers specifically to query or insert timeout
static void uvHandleActivityTimeout(uv_timer_t* handle) {
// impl later
SRpcConn* conn = handle->data;
}
static void uvProcessData(SRpcConn* pConn) {
SRecvInfo info;
SRecvInfo* p = &info;
SConnBuffer* pBuf = &pConn->connBuf;
p->msg = pBuf->buf + RPC_RESERVE_SIZE;
p->msgLen = pBuf->len;
p->ip = 0;
p->port = 0;
p->shandle = pConn->shandle; //
p->thandle = pConn;
p->chandle = NULL;
//
SRpcHead* pHead = (SRpcHead*)p->msg;
assert(rpcIsReq(pHead->msgType));
SRpcInfo* pRpc = (SRpcInfo*)p->shandle;
pConn->ahandle = (void*)pHead->ahandle;
// auth here
int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen);
if (code != 0) {
terrno = code;
return;
}
pHead->code = htonl(pHead->code);
SRpcMsg rpcMsg;
pHead = rpcDecompressRpcMsg(pHead);
rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen);
rpcMsg.pCont = pHead->content;
rpcMsg.msgType = pHead->msgType;
rpcMsg.code = pHead->code;
rpcMsg.ahandle = pConn->ahandle;
rpcMsg.handle = pConn;
(*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime, 0);
// auth
// validate msg type
}
void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
// opt
SRpcConn* ctx = cli->data;
SConnBuffer* pBuf = &ctx->connBuf;
if (nread > 0) {
pBuf->len += nread;
if (isReadAll(pBuf)) {
tDebug("alread read complete packet");
uvProcessData(ctx);
} else {
tDebug("read half packet, continue to read");
}
return;
}
if (terrno != 0) {
// handle err code
}
if (nread != UV_EOF) {
tDebug("Read error %s\n", uv_err_name(nread));
}
uv_close((uv_handle_t*)cli, uvConnDestroy);
}
void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
buf->base = malloc(sizeof(char));
buf->len = 2;
}
void uvOnTimeoutCb(uv_timer_t* handle) {
// opt
tDebug("time out");
}
void uvOnWriteCb(uv_write_t* req, int status) {
SRpcConn* conn = req->data;
if (status == 0) {
tDebug("data already was written on stream");
} else {
connDestroy(conn);
}
// opt
}
void uvWorkerAsyncCb(uv_async_t* handle) {
SThreadObj* pThrd = container_of(handle, SThreadObj, workerAsync);
SRpcConn* conn = NULL;
// opt later
pthread_mutex_lock(&pThrd->connMtx);
if (!QUEUE_IS_EMPTY(&pThrd->conn)) {
queue* head = QUEUE_HEAD(&pThrd->conn);
conn = QUEUE_DATA(head, SRpcConn, queue);
QUEUE_REMOVE(&conn->queue);
}
pthread_mutex_unlock(&pThrd->connMtx);
if (conn == NULL) {
tError("except occurred, do nothing");
return;
}
uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len);
uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb);
}
void uvOnAcceptCb(uv_stream_t* stream, int status) {
if (status == -1) {
return;
}
SServerObj* pObj = container_of(stream, SServerObj, server);
uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t));
uv_tcp_init(pObj->loop, cli);
if (uv_accept(stream, (uv_stream_t*)cli) == 0) {
uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t));
uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify));
pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread;
tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx);
uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb);
} else {
uv_close((uv_handle_t*)cli, NULL);
}
}
void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
tDebug("connection coming");
if (nread < 0) {
if (nread != UV_EOF) {
tError("read error %s", uv_err_name(nread));
}
// TODO(log other failure reason)
uv_close((uv_handle_t*)q, NULL);
return;
}
// free memory allocated by
assert(nread == strlen(notify));
assert(buf->base[0] == notify[0]);
free(buf->base);
SThreadObj* pThrd = q->data;
uv_pipe_t* pipe = (uv_pipe_t*)q;
if (!uv_pipe_pending_count(pipe)) {
tError("No pending count");
return;
}
uv_handle_type pending = uv_pipe_pending_type(pipe);
assert(pending == UV_TCP);
SRpcConn* pConn = connCreate();
pConn->shandle = pThrd->shandle;
/* init conn timer*/
pConn->pTimer = malloc(sizeof(uv_timer_t));
uv_timer_init(pThrd->loop, pConn->pTimer);
pConn->pTimer->data = pConn;
pConn->hostThread = pThrd;
pConn->pWorkerAsync = pThrd->workerAsync; // thread safty
// init client handle
pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t));
uv_tcp_init(pThrd->loop, pConn->pTcp);
pConn->pTcp->data = pConn;
// init write request, just
pConn->pWriter = calloc(1, sizeof(uv_write_t));
pConn->pWriter->data = pConn;
if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) {
uv_os_fd_t fd;
uv_fileno((const uv_handle_t*)pConn->pTcp, &fd);
tDebug("new connection created: %d", fd);
uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb);
} else {
connDestroy(pConn);
}
}
void* acceptThread(void* arg) {
// opt
SServerObj* srv = (SServerObj*)arg;
uv_tcp_init(srv->loop, &srv->server);
struct sockaddr_in bind_addr;
uv_ip4_addr("0.0.0.0", srv->port, &bind_addr);
uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0);
int err = 0;
if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) {
tError("Listen error %s\n", uv_err_name(err));
return NULL;
}
uv_run(srv->loop, UV_RUN_DEFAULT);
}
void* workerThread(void* arg) {
SThreadObj* pThrd = (SThreadObj*)arg;
pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
uv_loop_init(pThrd->loop);
uv_pipe_init(pThrd->loop, pThrd->pipe, 1);
uv_pipe_open(pThrd->pipe, pThrd->fd);
pThrd->pipe->data = pThrd;
QUEUE_INIT(&pThrd->conn);
pThrd->workerAsync = malloc(sizeof(uv_async_t));
uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb);
uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb);
uv_run(pThrd->loop, UV_RUN_DEFAULT);
}
static SRpcConn* connCreate() {
SRpcConn* pConn = (SRpcConn*)calloc(1, sizeof(SRpcConn));
return pConn;
}
static void connDestroy(SRpcConn* conn) {
if (conn == NULL) {
return;
}
uv_timer_stop(conn->pTimer);
free(conn->pTimer);
uv_close((uv_handle_t*)conn->pTcp, NULL);
free(conn->connBuf.buf);
free(conn->pTcp);
free(conn->pWriter);
free(conn);
// handle
}
static void uvConnDestroy(uv_handle_t* handle) {
SRpcConn* conn = handle->data;
connDestroy(conn);
}
void* rpcOpen(const SRpcInit* pInit) {
SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo));
if (pRpc == NULL) {
return NULL;
}
if (pInit->label) {
tstrncpy(pRpc->label, pInit->label, strlen(pInit->label));
}
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
pRpc->tcphandle = taosInitServer(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc);
return pRpc;
}
void rpcClose(void* arg) { return; }
void* rpcMallocCont(int contLen) { return NULL; }
void rpcFreeCont(void* cont) { return; }
void* rpcReallocCont(void* ptr, int contLen) { return NULL; }
void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) {
// impl later
return;
}
void rpcSendResponse(const SRpcMsg* pMsg) {
SRpcConn* pConn = pMsg->handle;
SThreadObj* pThrd = pConn->hostThread;
// opt later
pthread_mutex_lock(&pThrd->connMtx);
QUEUE_PUSH(&pThrd->conn, &pConn->queue);
pthread_mutex_unlock(&pThrd->connMtx);
uv_async_send(pConn->pWorkerAsync);
}
void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {}
int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; }
void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; }
int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; }
void rpcCancelRequest(int64_t rid) { return; }
static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) {
T_MD5_CTX context;
int ret = -1;
tMD5Init(&context);
tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN);
tMD5Update(&context, (uint8_t*)pMsg, msgLen);
tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN);
tMD5Final(&context);
if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0;
return ret;
}
static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) {
T_MD5_CTX context;
tMD5Init(&context);
tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN);
tMD5Update(&context, (uint8_t*)pMsg, msgLen);
tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN);
tMD5Final(&context);
memcpy(pAuth, context.digest, sizeof(context.digest));
}
static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen) {
SRpcHead* pHead = (SRpcHead*)msg;
if (pConn->spi && pConn->secured == 0) {
// add auth part
pHead->spi = pConn->spi;
SRpcDigest* pDigest = (SRpcDigest*)(msg + msgLen);
pDigest->timeStamp = htonl(taosGetTimestampSec());
msgLen += sizeof(SRpcDigest);
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret);
} else {
pHead->spi = 0;
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
}
return msgLen;
}
static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) {
SRpcHead* pHead = rpcHeadFromCont(pCont);
int32_t finalLen = 0;
int overhead = sizeof(SRpcComp);
if (!NEEDTO_COMPRESSS_MSG(contLen)) {
return contLen;
}
char* buf = malloc(contLen + overhead + 8); // 8 extra bytes
if (buf == NULL) {
tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen);
return contLen;
}
int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead);
tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead);
/*
* only the compressed size is less than the value of contLen - overhead, the compression is applied
* The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message
*/
if (compLen > 0 && compLen < contLen - overhead) {
SRpcComp* pComp = (SRpcComp*)pCont;
pComp->reserved = 0;
pComp->contLen = htonl(contLen);
memcpy(pCont + overhead, buf, compLen);
pHead->comp = 1;
tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen);
finalLen = compLen + overhead;
} else {
finalLen = contLen;
}
free(buf);
return finalLen;
}
static SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) {
int overhead = sizeof(SRpcComp);
SRpcHead* pNewHead = NULL;
uint8_t* pCont = pHead->content;
SRpcComp* pComp = (SRpcComp*)pHead->content;
if (pHead->comp) {
// decompress the content
assert(pComp->reserved == 0);
int contLen = htonl(pComp->contLen);
// prepare the temporary buffer to decompress message
char* temp = (char*)malloc(contLen + RPC_MSG_OVERHEAD);
pNewHead = (SRpcHead*)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext
if (pNewHead) {
int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead;
int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char*)pNewHead->content, compLen, contLen);
assert(origLen == contLen);
memcpy(pNewHead, pHead, sizeof(SRpcHead));
pNewHead->msgLen = rpcMsgLenFromCont(origLen);
/// rpcFreeMsg(pHead); // free the compressed message buffer
pHead = pNewHead;
tTrace("decomp malloc mem:%p", temp);
} else {
tError("failed to allocate memory to decompress msg, contLen:%d", contLen);
}
}
return pHead;
}
int32_t rpcInit(void) {
// impl later
return -1;
}
void rpcCleanup(void) {
// impl later
return;
}
#endif
source/libs/transport/test/CMakeLists.txt 0 → 100644
浏览文件 @ ae2d77e9
add_executable(transportTest "")
target_sources(transportTest
PRIVATE
"transportTests.cc"
)
target_include_directories(transportTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/transport"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (transportTest
os
util
common
gtest_main
transport
)
source/libs/transport/test/transportTests.cc 0 → 100644
浏览文件 @ ae2d77e9
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <chrono>
#include <iostream>
#include <string>
#include <thread>
#include "transportInt.h"
#include "trpc.h"
using namespace std;
int main() {
SRpcInit init = {.localPort = 6030, .label = "rpc", .numOfThreads = 5};
void* p = rpcOpen(&init);
while (1) {
std::cout << "cron task" << std::endl;
std::this_thread::sleep_for(std::chrono::milliseconds(10 * 1000));
}
}
source/libs/wal/src/walMeta.c
浏览文件 @ ae2d77e9
......@@ -36,7 +36,7 @@ void* tmemmem(char* haystack, int hlen, char* needle, int nlen) {
char* limit;
if (nlen == 0 || hlen < nlen) {
return false;
return NULL;
}
limit = haystack + hlen - nlen + 1;
......@@ -54,10 +54,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
ASSERT(pWal->fileInfoSet != NULL);
int sz = taosArrayGetSize(pWal->fileInfoSet);
ASSERT(sz > 0);
#if 0
for (int i = 0; i < sz; i++) {
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
}
#endif
SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz-1);
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr);
......@@ -143,8 +145,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
SWalFileInfo fileInfo;
memset(&fileInfo, -1, sizeof(SWalFileInfo));
sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer);
//get lastVer
//get size
taosArrayPush(pLogInfoArray, &fileInfo);
}
}
......@@ -158,15 +158,29 @@ int walCheckAndRepairMeta(SWal* pWal) {
oldSz = taosArrayGetSize(pWal->fileInfoSet);
}
int newSz = taosArrayGetSize(pLogInfoArray);
// case 1. meta file not exist / cannot be parsed
if (oldSz < newSz) {
if (oldSz > newSz) {
taosArrayPopFrontBatch(pWal->fileInfoSet, oldSz - newSz);
} else if (oldSz < newSz) {
for (int i = oldSz; i < newSz; i++) {
SWalFileInfo *pFileInfo = taosArrayGet(pLogInfoArray, i);
taosArrayPush(pWal->fileInfoSet, pFileInfo);
}
}
taosArrayDestroy(pLogInfoArray);
pWal->writeCur = newSz - 1;
pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pLogInfoArray, 0))->firstVer;
if (newSz > 0) {
pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1);
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr);
struct stat statbuf;
stat(fnameStr, &statbuf);
if (oldSz != newSz || pLastFileInfo->fileSize != statbuf.st_size) {
pLastFileInfo->fileSize = statbuf.st_size;
pWal->vers.lastVer = walScanLogGetLastVer(pWal);
((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer;
ASSERT(pWal->vers.lastVer != -1);
......@@ -177,41 +191,18 @@ int walCheckAndRepairMeta(SWal* pWal) {
return -1;
}
}
// case 2. versions in meta not match log
// or some log not included in meta
// (e.g. program killed)
//
// case 3. other corrupt cases
//
#if 0
int sz = taosArrayGetSize(pLogInfoArray);
for (int i = 0; i < sz; i++) {
SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, i);
if (i == 0 && pFileInfo->firstVer != walGetFirstVer(pWal)) {
//repair
}
if (i > 0) {
SWalFileInfo* pLastFileInfo = taosArrayGet(pLogInfoArray, i-1);
if (pLastFileInfo->lastVer != pFileInfo->firstVer) {
}
}
}
#endif
//TODO: set fileSize and lastVer if necessary
// get last version of this file
//
// rebuild meta
taosArrayDestroy(pLogInfoArray);
return 0;
}
int walCheckAndRepairIdx(SWal* pWal) {
// iterate all idx files
// check first and last entry of each idx file valid
// TODO: iterate all log files
// if idx not found, scan log and write idx
// if found, check complete by first and last entry of each idx file
// if idx incomplete, binary search last valid entry, and then build other part
return 0;
}
......
source/util/src/terror.c
浏览文件 @ ae2d77e9
......@@ -307,6 +307,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, "Database suspended")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, "Database is syncing")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TSDB_STATE, "Invalid tsdb state")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists")
// tsdb
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID")
......
tests/test/c/create_table.c
浏览文件 @ ae2d77e9
......@@ -30,6 +30,7 @@ int32_t createTable = 1;
int32_t insertData = 0;
int32_t batchNum = 100;
int32_t numOfVgroups = 2;
int32_t showTablesFlag = 0;
typedef struct {
int64_t tableBeginIndex;
......@@ -45,88 +46,9 @@ typedef struct {
pthread_t thread;
} SThreadInfo;
void parseArgument(int32_t argc, char *argv[]);
void *threadFunc(void *param);
void createDbAndStb();
int32_t main(int32_t argc, char *argv[]) {
parseArgument(argc, argv);
createDbAndStb();
pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo));
//int64_t numOfTablesPerThread = numOfTables / numOfThreads;
//numOfTables = numOfTablesPerThread * numOfThreads;
if (numOfThreads < 1) {
numOfThreads = 1;
}
int64_t a = numOfTables / numOfThreads;
if (a < 1) {
numOfThreads = numOfTables;
a = 1;
}
int64_t b = 0;
b = numOfTables % numOfThreads;
int64_t tableFrom = 0;
for (int32_t i = 0; i < numOfThreads; ++i) {
pInfo[i].tableBeginIndex = tableFrom;
pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1;
tableFrom = pInfo[i].tableEndIndex + 1;
pInfo[i].threadIndex = i;
pInfo[i].minDelay = INT64_MAX;
strcpy(pInfo[i].dbName, dbName);
strcpy(pInfo[i].stbName, stbName);
pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i));
}
taosMsleep(300);
for (int32_t i = 0; i < numOfThreads; i++) {
pthread_join(pInfo[i].thread, NULL);
}
int64_t maxDelay = 0;
int64_t minDelay = INT64_MAX;
float createTableSpeed = 0;
for (int32_t i = 0; i < numOfThreads; ++i) {
createTableSpeed += pInfo[i].createTableSpeed;
if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay;
if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay;
}
float insertDataSpeed = 0;
for (int32_t i = 0; i < numOfThreads; ++i) {
insertDataSpeed += pInfo[i].insertDataSpeed;
}
pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s",
GREEN,
numOfTables,
createTableSpeed,
numOfThreads,
maxDelay,
minDelay,
NC);
if (insertData) {
pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed,
numOfThreads, NC);
}
pthread_attr_destroy(&thattr);
free(pInfo);
}
//void parseArgument(int32_t argc, char *argv[]);
//void *threadFunc(void *param);
//void createDbAndStb();
void createDbAndStb() {
pPrint("start to create db and stable");
......@@ -188,6 +110,62 @@ void printInsertProgress(SThreadInfo *pInfo, int64_t t) {
totalTables, seconds, speed);
}
static int64_t getResult(TAOS_RES *tres) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
}
int num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int precision = taos_result_precision(tres);
int64_t numOfRows = 0;
do {
numOfRows++;
row = taos_fetch_row(tres);
} while (row != NULL);
return numOfRows;
}
void showTables() {
pPrint("start to show tables");
char qstr[32];
TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
if (con == NULL) {
pError("failed to connect to DB, reason:%s", taos_errstr(NULL));
exit(1);
}
sprintf(qstr, "use %s", dbName);
TAOS_RES *pRes = taos_query(con, qstr);
int code = taos_errno(pRes);
if (code != 0) {
pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
exit(1);
}
taos_free_result(pRes);
sprintf(qstr, "show tables");
pRes = taos_query(con, qstr);
code = taos_errno(pRes);
if (code != 0) {
pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
exit(0);
}
int64_t totalTableNum = getResult(pRes);
taos_free_result(pRes);
pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC);
taos_close(con);
}
void *threadFunc(void *param) {
SThreadInfo *pInfo = (SThreadInfo *)param;
char *qstr = malloc(2000 * 1000);
......@@ -209,13 +187,19 @@ void *threadFunc(void *param) {
int64_t curMs = 0;
int64_t beginMs = taosGetTimestampMs();
pInfo->startMs = beginMs;
for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
int64_t batch = (pInfo->tableEndIndex - t);
batch = MIN(batch, batchNum);
int64_t t = pInfo->tableBeginIndex;
for (; t <= pInfo->tableEndIndex;) {
//int64_t batch = (pInfo->tableEndIndex - t);
//batch = MIN(batch, batchNum);
int32_t len = sprintf(qstr, "create table");
for (int32_t i = 0; i < batch; ++i) {
len += sprintf(qstr + len, " t%" PRId64 " using %s tags(%" PRId64 ")", t + i, stbName, t + i);
for (int32_t i = 0; i < batchNum;) {
len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t);
t++;
i++;
if (t > pInfo->tableEndIndex) {
break;
}
}
int64_t startTs = taosGetTimestampUs();
......@@ -234,11 +218,11 @@ void *threadFunc(void *param) {
curMs = taosGetTimestampMs();
if (curMs - beginMs > 10000) {
beginMs = curMs;
//printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t);
printCreateProgress(pInfo, t);
}
t += (batch - 1);
}
printCreateProgress(pInfo, pInfo->tableEndIndex);
printCreateProgress(pInfo, t);
}
if (insertData) {
......@@ -298,6 +282,8 @@ void printHelp() {
printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData);
printf("%s%s\n", indent, "-b");
printf("%s%s%s%d\n", indent, indent, "batchNum, default is ", batchNum);
printf("%s%s\n", indent, "-w");
printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag);
exit(EXIT_SUCCESS);
}
......@@ -325,6 +311,8 @@ void parseArgument(int32_t argc, char *argv[]) {
insertData = atoi(argv[++i]);
} else if (strcmp(argv[i], "-b") == 0) {
batchNum = atoi(argv[++i]);
} else if (strcmp(argv[i], "-w") == 0) {
showTablesFlag = atoi(argv[++i]);
} else {
}
}
......@@ -338,6 +326,93 @@ void parseArgument(int32_t argc, char *argv[]) {
pPrint("%s createTable:%d %s", GREEN, createTable, NC);
pPrint("%s insertData:%d %s", GREEN, insertData, NC);
pPrint("%s batchNum:%d %s", GREEN, batchNum, NC);
pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC);
pPrint("%s start create table performace test %s", GREEN, NC);
}
int32_t main(int32_t argc, char *argv[]) {
parseArgument(argc, argv);
if (showTablesFlag) {
showTables();
return 0;
}
createDbAndStb();
pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo));
//int64_t numOfTablesPerThread = numOfTables / numOfThreads;
//numOfTables = numOfTablesPerThread * numOfThreads;
if (numOfThreads < 1) {
numOfThreads = 1;
}
int64_t a = numOfTables / numOfThreads;
if (a < 1) {
numOfThreads = numOfTables;
a = 1;
}
int64_t b = 0;
b = numOfTables % numOfThreads;
int64_t tableFrom = 0;
for (int32_t i = 0; i < numOfThreads; ++i) {
pInfo[i].tableBeginIndex = tableFrom;
pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1;
tableFrom = pInfo[i].tableEndIndex + 1;
pInfo[i].threadIndex = i;
pInfo[i].minDelay = INT64_MAX;
strcpy(pInfo[i].dbName, dbName);
strcpy(pInfo[i].stbName, stbName);
pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i));
}
taosMsleep(300);
for (int32_t i = 0; i < numOfThreads; i++) {
pthread_join(pInfo[i].thread, NULL);
}
int64_t maxDelay = 0;
int64_t minDelay = INT64_MAX;
float createTableSpeed = 0;
for (int32_t i = 0; i < numOfThreads; ++i) {
createTableSpeed += pInfo[i].createTableSpeed;
if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay;
if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay;
}
float insertDataSpeed = 0;
for (int32_t i = 0; i < numOfThreads; ++i) {
insertDataSpeed += pInfo[i].insertDataSpeed;
}
pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s",
GREEN,
numOfTables,
createTableSpeed,
numOfThreads,
maxDelay,
minDelay,
NC);
if (insertData) {
pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed,
numOfThreads, NC);
}
pthread_attr_destroy(&thattr);
free(pInfo);
}
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或