提交 eb573b7f 编写于 作者: H Haojun Liao

Merge remote-tracking branch 'origin/3.0_query_integrate' into 3.0_query_integrate

add_executable(simulate_vnode "simulate_vnode.c") add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode craft lz4 uv_a) target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
\ No newline at end of file \ No newline at end of file
...@@ -3,4 +3,4 @@ target_sources(singleNode ...@@ -3,4 +3,4 @@ target_sources(singleNode
PRIVATE PRIVATE
"singleNode.c" "singleNode.c"
) )
target_link_libraries(singleNode traft lz4 uv_a) target_link_libraries(singleNode PUBLIC traft lz4 uv_a)
...@@ -25,24 +25,6 @@ ...@@ -25,24 +25,6 @@
extern "C" { extern "C" {
#endif #endif
// typedef struct STimeWindow {
// TSKEY skey;
// TSKEY ekey;
// } STimeWindow;
// typedef struct {
// int32_t dataLen;
// char name[TSDB_TABLE_FNAME_LEN];
// char *data;
// } STagData;
// typedef struct SSchema {
// uint8_t type;
// char name[TSDB_COL_NAME_LEN];
// int16_t colId;
// int16_t bytes;
// } SSchema;
enum { enum {
TMQ_CONF__RESET_OFFSET__LATEST = -1, TMQ_CONF__RESET_OFFSET__LATEST = -1,
TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
...@@ -50,7 +32,8 @@ enum { ...@@ -50,7 +32,8 @@ enum {
}; };
enum { enum {
TMQ_MSG_TYPE__POLL_RSP = 0, TMQ_MSG_TYPE__DUMMY = 0,
TMQ_MSG_TYPE__POLL_RSP,
TMQ_MSG_TYPE__EP_RSP, TMQ_MSG_TYPE__EP_RSP,
}; };
...@@ -285,4 +268,4 @@ typedef struct SSessionWindow { ...@@ -285,4 +268,4 @@ typedef struct SSessionWindow {
} }
#endif #endif
#endif /*_TD_COMMON_DEF_H_*/ #endif /*_TD_COMMON_DEF_H_*/
...@@ -70,7 +70,7 @@ typedef uint16_t tmsg_t; ...@@ -70,7 +70,7 @@ typedef uint16_t tmsg_t;
typedef enum { typedef enum {
HEARTBEAT_TYPE_MQ = 0, HEARTBEAT_TYPE_MQ = 0,
HEARTBEAT_TYPE_QUERY = 1, HEARTBEAT_TYPE_QUERY,
// types can be added here // types can be added here
// //
HEARTBEAT_TYPE_MAX HEARTBEAT_TYPE_MAX
...@@ -140,6 +140,8 @@ typedef enum _mgmt_table { ...@@ -140,6 +140,8 @@ typedef enum _mgmt_table {
#define TSDB_KILL_MSG_LEN 30 #define TSDB_KILL_MSG_LEN 30
#define TSDB_TABLE_NUM_UNIT 100000
#define TSDB_VN_READ_ACCCESS ((char)0x1) #define TSDB_VN_READ_ACCCESS ((char)0x1)
#define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_WRITE_ACCCESS ((char)0x2)
#define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS)
...@@ -169,6 +171,7 @@ typedef struct { ...@@ -169,6 +171,7 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
int64_t dbId; int64_t dbId;
int32_t vgVersion; int32_t vgVersion;
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
} SBuildUseDBInput; } SBuildUseDBInput;
typedef struct SField { typedef struct SField {
...@@ -570,6 +573,7 @@ typedef struct { ...@@ -570,6 +573,7 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
int64_t dbId; int64_t dbId;
int32_t vgVersion; int32_t vgVersion;
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
} SUseDbReq; } SUseDbReq;
int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq);
...@@ -675,6 +679,11 @@ typedef struct { ...@@ -675,6 +679,11 @@ typedef struct {
int64_t totalStorage; int64_t totalStorage;
int64_t compStorage; int64_t compStorage;
int64_t pointsWritten; int64_t pointsWritten;
int64_t numOfSelectReqs;
int64_t numOfInsertReqs;
int64_t numOfInsertSuccessReqs;
int64_t numOfBatchInsertReqs;
int64_t numOfBatchInsertSuccessReqs;
} SVnodeLoad; } SVnodeLoad;
typedef struct { typedef struct {
...@@ -795,8 +804,10 @@ typedef struct SVgroupInfo { ...@@ -795,8 +804,10 @@ typedef struct SVgroupInfo {
uint32_t hashBegin; uint32_t hashBegin;
uint32_t hashEnd; uint32_t hashEnd;
SEpSet epset; SEpSet epset;
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
} SVgroupInfo; } SVgroupInfo;
typedef struct { typedef struct {
int32_t numOfVgroups; int32_t numOfVgroups;
SVgroupInfo vgroups[]; SVgroupInfo vgroups[];
...@@ -1841,6 +1852,203 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) ...@@ -1841,6 +1852,203 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW)
} }
return buf; return buf;
} }
typedef enum {
TD_TIME_UNIT_UNKNOWN = -1,
TD_TIME_UNIT_YEAR = 0,
TD_TIME_UNIT_SEASON = 1,
TD_TIME_UNIT_MONTH = 2,
TD_TIME_UNIT_WEEK = 3,
TD_TIME_UNIT_DAY = 4,
TD_TIME_UNIT_HOUR = 5,
TD_TIME_UNIT_MINUTE = 6,
TD_TIME_UNIT_SEC = 7,
TD_TIME_UNIT_MILLISEC = 8,
TD_TIME_UNIT_MICROSEC = 9,
TD_TIME_UNIT_NANOSEC = 10
} ETDTimeUnit;
typedef struct {
uint8_t version; // for compatibility
uint8_t intervalUnit;
uint8_t slidingUnit;
char indexName[TSDB_INDEX_NAME_LEN + 1];
col_id_t numOfColIds;
uint16_t numOfFuncIds;
uint64_t tableUid; // super/common table uid
int64_t interval;
int64_t sliding;
col_id_t* colIds; // sorted column ids
uint16_t* funcIds; // sorted sma function ids
} STSma; // Time-range-wise SMA
typedef struct {
int8_t msgType; // 0 create, 1 recreate
STSma tSma;
STimeWindow window;
} SCreateTSmaMsg;
typedef struct {
STimeWindow window;
char indexName[TSDB_INDEX_NAME_LEN + 1];
} SDropTSmaMsg;
typedef struct {
STimeWindow tsWindow; // [skey, ekey]
uint64_t tableUid; // sub/common table uid
int32_t numOfBlocks; // number of sma blocks for each column, total number is numOfBlocks*numOfColId
int32_t dataLen; // total data length
col_id_t* colIds; // e.g. 2,4,9,10
col_id_t numOfColIds; // e.g. 4
char data[]; // the sma blocks
} STSmaData;
// TODO: move to the final location afte schema of STSma/STSmaData defined
static FORCE_INLINE void tdDestroySmaData(STSmaData* pSmaData) {
if (pSmaData) {
if (pSmaData->colIds) {
tfree(pSmaData->colIds);
}
tfree(pSmaData);
}
}
// RSma: Time-range-wise Rollup SMA
// TODO: refactor when rSma grammar defined finally =>
typedef struct {
int64_t interval;
int32_t retention; // unit: day
uint16_t days; // unit: day
int8_t intervalUnit;
} SSmaParams;
// TODO: refactor when rSma grammar defined finally <=
typedef struct {
// TODO: refactor to use the real schema =>
STSma tsma;
float xFilesFactor;
SArray* smaParams; // SSmaParams
// TODO: refactor to use the real schema <=
} SRSma;
typedef struct {
uint32_t number;
STSma* tSma;
} STSmaWrapper;
static FORCE_INLINE void tdDestroyTSma(STSma* pSma, bool releaseSelf) {
if (pSma) {
tfree(pSma->colIds);
tfree(pSma->funcIds);
if (releaseSelf) {
free(pSma);
}
}
}
static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool releaseSelf) {
if (pSW) {
if (pSW->tSma) {
for (uint32_t i = 0; i < pSW->number; ++i) {
tdDestroyTSma(pSW->tSma + i, false);
}
tfree(pSW->tSma);
}
if (releaseSelf) {
free(pSW);
}
}
}
static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
int32_t tlen = 0;
tlen += taosEncodeFixedU8(buf, pSma->version);
tlen += taosEncodeFixedU8(buf, pSma->intervalUnit);
tlen += taosEncodeFixedU8(buf, pSma->slidingUnit);
tlen += taosEncodeString(buf, pSma->indexName);
tlen += taosEncodeFixedU16(buf, pSma->numOfColIds);
tlen += taosEncodeFixedU16(buf, pSma->numOfFuncIds);
tlen += taosEncodeFixedU64(buf, pSma->tableUid);
tlen += taosEncodeFixedI64(buf, pSma->interval);
tlen += taosEncodeFixedI64(buf, pSma->sliding);
for (col_id_t i = 0; i < pSma->numOfColIds; ++i) {
tlen += taosEncodeFixedU16(buf, *(pSma->colIds + i));
}
for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) {
tlen += taosEncodeFixedU16(buf, *(pSma->funcIds + i));
}
return tlen;
}
static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) {
int32_t tlen = 0;
tlen += taosEncodeFixedU32(buf, pSW->number);
for (uint32_t i = 0; i < pSW->number; ++i) {
tlen += tEncodeTSma(buf, pSW->tSma + i);
}
return tlen;
}
static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
buf = taosDecodeFixedU8(buf, &pSma->version);
buf = taosDecodeFixedU8(buf, &pSma->intervalUnit);
buf = taosDecodeFixedU8(buf, &pSma->slidingUnit);
buf = taosDecodeStringTo(buf, pSma->indexName);
buf = taosDecodeFixedU16(buf, &pSma->numOfColIds);
buf = taosDecodeFixedU16(buf, &pSma->numOfFuncIds);
buf = taosDecodeFixedU64(buf, &pSma->tableUid);
buf = taosDecodeFixedI64(buf, &pSma->interval);
buf = taosDecodeFixedI64(buf, &pSma->sliding);
if (pSma->numOfColIds > 0) {
pSma->colIds = (col_id_t*)calloc(pSma->numOfColIds, sizeof(STSma));
if (pSma->colIds == NULL) {
return NULL;
}
for (uint16_t i = 0; i < pSma->numOfColIds; ++i) {
buf = taosDecodeFixedU16(buf, pSma->colIds + i);
}
} else {
pSma->colIds = NULL;
}
if (pSma->numOfFuncIds > 0) {
pSma->funcIds = (uint16_t*)calloc(pSma->numOfFuncIds, sizeof(STSma));
if (pSma->funcIds == NULL) {
return NULL;
}
for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) {
buf = taosDecodeFixedU16(buf, pSma->funcIds + i);
}
} else {
pSma->funcIds = NULL;
}
return buf;
}
static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
buf = taosDecodeFixedU32(buf, &pSW->number);
pSW->tSma = (STSma*)calloc(pSW->number, sizeof(STSma));
if (pSW->tSma == NULL) {
return NULL;
}
for (uint32_t i = 0; i < pSW->number; ++i) {
if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) {
for (uint32_t j = i; j >= 0; --i) {
tdDestroyTSma(pSW->tSma + j, false);
}
free(pSW->tSma);
return NULL;
}
}
return buf;
}
typedef struct { typedef struct {
int64_t uid; int64_t uid;
......
...@@ -184,6 +184,9 @@ enum { ...@@ -184,6 +184,9 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
// Requests handled by QNODE // Requests handled by QNODE
TD_NEW_MSG_SEG(TDMT_QND_MSG) TD_NEW_MSG_SEG(TDMT_QND_MSG)
......
...@@ -118,6 +118,8 @@ typedef struct { ...@@ -118,6 +118,8 @@ typedef struct {
} SKvRow; } SKvRow;
typedef struct { typedef struct {
/// timestamp
TSKEY ts;
union { union {
/// union field for encode and decode /// union field for encode and decode
uint32_t info; uint32_t info;
...@@ -138,8 +140,6 @@ typedef struct { ...@@ -138,8 +140,6 @@ typedef struct {
uint32_t len; uint32_t len;
/// row version /// row version
uint64_t ver; uint64_t ver;
/// timestamp
TSKEY ts;
/// the inline data, maybe a tuple or a k-v tuple /// the inline data, maybe a tuple or a k-v tuple
char data[]; char data[];
} STSRow; } STSRow;
...@@ -173,7 +173,7 @@ typedef struct { ...@@ -173,7 +173,7 @@ typedef struct {
#define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_DATA(r) ((r)->data)
#define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_LEN(r) ((r)->len)
#define TD_ROW_KEY(r) ((r)->ts) #define TD_ROW_KEY(r) ((r)->ts)
#define TD_ROW_KEY_ADDR(r) POINTER_SHIFT((r), 16) #define TD_ROW_KEY_ADDR(r) (r)
// N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
// (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
......
...@@ -74,9 +74,9 @@ typedef struct SDbVgVersion { ...@@ -74,9 +74,9 @@ typedef struct SDbVgVersion {
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
int64_t dbId; int64_t dbId;
int32_t vgVersion; int32_t vgVersion;
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
} SDbVgVersion; } SDbVgVersion;
int32_t catalogInit(SCatalogCfg *cfg); int32_t catalogInit(SCatalogCfg *cfg);
/** /**
...@@ -95,7 +95,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); ...@@ -95,7 +95,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle);
*/ */
void catalogFreeHandle(SCatalog* pCatalog); void catalogFreeHandle(SCatalog* pCatalog);
int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId); int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum);
/** /**
* Get a DB's all vgroup info. * Get a DB's all vgroup info.
......
...@@ -89,25 +89,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds); ...@@ -89,25 +89,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds);
*/ */
int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext); int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext);
/**
*
* Retrieve the actual results to fill the response message payload.
* Note that this function must be executed after qRetrieveQueryResultInfo is invoked.
*
* @param tinfo tinfo object
* @param pRsp response message
* @param contLen payload length
* @return
*/
//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
/**
* return the transporter context (RPC)
* @param tinfo
* @return
*/
void* qGetResultRetrieveMsg(qTaskInfo_t tinfo);
/** /**
* kill the ongoing query and free the query handle and corresponding resources automatically * kill the ongoing query and free the query handle and corresponding resources automatically
* @param tinfo qhandle * @param tinfo qhandle
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "tarray.h" #include "tarray.h"
#include "tdef.h" #include "tdef.h"
#include "tlog.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
...@@ -85,29 +86,26 @@ typedef struct { ...@@ -85,29 +86,26 @@ typedef struct {
typedef struct { typedef struct {
float uptime; // day float uptime; // day
float cpu_engine; double cpu_engine;
float cpu_system; double cpu_system;
float cpu_cores; float cpu_cores;
int64_t mem_engine; // KB int64_t mem_engine; // KB
int64_t mem_system; // KB int64_t mem_system; // KB
int64_t mem_total; // KB int64_t mem_total; // KB
float disk_engine; // GB int64_t disk_engine; // Byte
float disk_used; // GB int64_t disk_used; // Byte
float disk_total; // GB int64_t disk_total; // Byte
int64_t net_in; int64_t net_in; // bytes
int64_t net_out; int64_t net_out; // bytes
float io_read; int64_t io_read; // bytes
float io_write; int64_t io_write; // bytes
float io_read_disk; int64_t io_read_disk; // bytes
float io_write_disk; int64_t io_write_disk; // bytes
int32_t req_select; int64_t req_select;
float req_select_rate; int64_t req_insert;
int32_t req_insert; int64_t req_insert_success;
int32_t req_insert_success; int64_t req_insert_batch;
float req_insert_rate; int64_t req_insert_batch_success;
int32_t req_insert_batch;
int32_t req_insert_batch_success;
float req_insert_batch_rate;
int32_t errors; int32_t errors;
int32_t vnodes_num; int32_t vnodes_num;
int32_t masters; int32_t masters;
...@@ -116,7 +114,7 @@ typedef struct { ...@@ -116,7 +114,7 @@ typedef struct {
typedef struct { typedef struct {
char name[TSDB_FILENAME_LEN]; char name[TSDB_FILENAME_LEN];
int32_t level; int8_t level;
SDiskSize size; SDiskSize size;
} SMonDiskDesc; } SMonDiskDesc;
...@@ -126,12 +124,6 @@ typedef struct { ...@@ -126,12 +124,6 @@ typedef struct {
SMonDiskDesc tempdir; SMonDiskDesc tempdir;
} SMonDiskInfo; } SMonDiskInfo;
typedef struct {
int64_t ts;
int8_t level;
char content[MON_LOG_LEN];
} SMonLogItem;
typedef struct SMonInfo SMonInfo; typedef struct SMonInfo SMonInfo;
typedef struct { typedef struct {
...@@ -142,7 +134,7 @@ typedef struct { ...@@ -142,7 +134,7 @@ typedef struct {
int32_t monInit(const SMonCfg *pCfg); int32_t monInit(const SMonCfg *pCfg);
void monCleanup(); void monCleanup();
void monAddLogItem(SMonLogItem *pItem); void monRecordLog(int64_t ts, ELogLevel level, const char *content);
SMonInfo *monCreateMonitorInfo(); SMonInfo *monCreateMonitorInfo();
void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);
......
...@@ -202,6 +202,7 @@ typedef struct SSubplan { ...@@ -202,6 +202,7 @@ typedef struct SSubplan {
int32_t msgType; // message type for subplan, used to denote the send message type to vnode. int32_t msgType; // message type for subplan, used to denote the send message type to vnode.
int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner.
SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node
SQueryNodeStat execNodeStat; // only for scan subplan
SNodeList* pChildren; // the datasource subplan,from which to fetch the result SNodeList* pChildren; // the datasource subplan,from which to fetch the result
SNodeList* pParents; // the data destination subplan, get data from current subplan SNodeList* pParents; // the data destination subplan, get data from current subplan
SPhysiNode* pNode; // physical plan of current subplan SPhysiNode* pNode; // physical plan of current subplan
......
...@@ -35,7 +35,6 @@ enum { ...@@ -35,7 +35,6 @@ enum {
JOB_TASK_STATUS_CANCELLING, JOB_TASK_STATUS_CANCELLING,
JOB_TASK_STATUS_CANCELLED, JOB_TASK_STATUS_CANCELLED,
JOB_TASK_STATUS_DROPPING, JOB_TASK_STATUS_DROPPING,
JOB_TASK_STATUS_FREEING,
}; };
enum { enum {
...@@ -83,6 +82,7 @@ typedef struct STableMeta { ...@@ -83,6 +82,7 @@ typedef struct STableMeta {
typedef struct SDBVgInfo { typedef struct SDBVgInfo {
int32_t vgVersion; int32_t vgVersion;
int8_t hashMethod; int8_t hashMethod;
int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT
SHashObj *vgHash; //key:vgId, value:SVgroupInfo SHashObj *vgHash; //key:vgId, value:SVgroupInfo
} SDBVgInfo; } SDBVgInfo;
...@@ -133,6 +133,10 @@ typedef struct SQueryNodeAddr { ...@@ -133,6 +133,10 @@ typedef struct SQueryNodeAddr {
SEpSet epset; SEpSet epset;
} SQueryNodeAddr; } SQueryNodeAddr;
typedef struct SQueryNodeStat {
int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
} SQueryNodeStat;
int32_t initTaskQueue(); int32_t initTaskQueue();
int32_t cleanupTaskQueue(); int32_t cleanupTaskQueue();
...@@ -168,20 +172,18 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl ...@@ -168,20 +172,18 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl
extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize);
#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE
#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE #define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE
#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
#define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) #define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0)
#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) #define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0)
#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) #define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }} while(0)
#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -19,10 +19,12 @@ ...@@ -19,10 +19,12 @@
extern "C" { extern "C" {
#endif #endif
#include "tcommon.h"
#include "nodes.h"
typedef struct SFilterInfo SFilterInfo; typedef struct SFilterInfo SFilterInfo;
typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
enum { enum {
FLT_OPTION_NO_REWRITE = 1, FLT_OPTION_NO_REWRITE = 1,
FLT_OPTION_TIMESTAMP = 2, FLT_OPTION_TIMESTAMP = 2,
...@@ -34,7 +36,6 @@ typedef struct SFilterColumnParam{ ...@@ -34,7 +36,6 @@ typedef struct SFilterColumnParam{
SArray* pDataBlock; SArray* pDataBlock;
} SFilterColumnParam; } SFilterColumnParam;
extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);
......
...@@ -25,6 +25,7 @@ extern "C" { ...@@ -25,6 +25,7 @@ extern "C" {
typedef struct SSchedulerCfg { typedef struct SSchedulerCfg {
uint32_t maxJobNum; uint32_t maxJobNum;
int32_t maxNodeTableNum;
} SSchedulerCfg; } SSchedulerCfg;
typedef struct SQueryProfileSummary { typedef struct SQueryProfileSummary {
......
...@@ -21,7 +21,7 @@ extern "C" { ...@@ -21,7 +21,7 @@ extern "C" {
#endif #endif
#include <stdint.h> #include <stdint.h>
#include <tep.h> #include <tdatablock.h>
#include "taosdef.h" #include "taosdef.h"
#include "trpc.h" #include "trpc.h"
...@@ -138,6 +138,8 @@ typedef struct SSyncInfo { ...@@ -138,6 +138,8 @@ typedef struct SSyncInfo {
void* rpcClient; void* rpcClient;
int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
void* queue;
int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg);
} SSyncInfo; } SSyncInfo;
...@@ -147,13 +149,10 @@ typedef struct SSyncNode SSyncNode; ...@@ -147,13 +149,10 @@ typedef struct SSyncNode SSyncNode;
int32_t syncInit(); int32_t syncInit();
void syncCleanUp(); void syncCleanUp();
int64_t syncStart(const SSyncInfo* pSyncInfo); int64_t syncStart(const SSyncInfo* pSyncInfo);
void syncStop(int64_t rid); void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
// int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
ESyncState syncGetMyRole(int64_t rid); ESyncState syncGetMyRole(int64_t rid);
void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);
......
...@@ -24,43 +24,41 @@ ...@@ -24,43 +24,41 @@
extern "C" { extern "C" {
#endif #endif
extern int32_t wDebugFlag; #define wFatal(...) \
{ \
#define wFatal(...) \ if (wDebugFlag & DEBUG_FATAL) { \
{ \ taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
if (wDebugFlag & DEBUG_FATAL) { \ } \
taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
} \
} }
#define wError(...) \ #define wError(...) \
{ \ { \
if (wDebugFlag & DEBUG_ERROR) { \ if (wDebugFlag & DEBUG_ERROR) { \
taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \ taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \ } \
} }
#define wWarn(...) \ #define wWarn(...) \
{ \ { \
if (wDebugFlag & DEBUG_WARN) { \ if (wDebugFlag & DEBUG_WARN) { \
taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \ taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \ } \
} }
#define wInfo(...) \ #define wInfo(...) \
{ \ { \
if (wDebugFlag & DEBUG_INFO) { \ if (wDebugFlag & DEBUG_INFO) { \
taosPrintLog("WAL ", 255, __VA_ARGS__); \ taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \
} \ } \
} }
#define wDebug(...) \ #define wDebug(...) \
{ \ { \
if (wDebugFlag & DEBUG_DEBUG) { \ if (wDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \
} \ } \
} }
#define wTrace(...) \ #define wTrace(...) \
{ \ { \
if (wDebugFlag & DEBUG_TRACE) { \ if (wDebugFlag & DEBUG_TRACE) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \
} \ } \
} }
#define WAL_HEAD_VER 0 #define WAL_HEAD_VER 0
......
...@@ -44,6 +44,7 @@ extern "C" { ...@@ -44,6 +44,7 @@ extern "C" {
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/utsname.h> #include <sys/utsname.h>
#include <sys/param.h>
#include <unistd.h> #include <unistd.h>
#include <wchar.h> #include <wchar.h>
#include <wctype.h> #include <wctype.h>
......
...@@ -22,6 +22,7 @@ extern "C" { ...@@ -22,6 +22,7 @@ extern "C" {
#include "osSocket.h" #include "osSocket.h"
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID #define open OPEN_FUNC_TAOS_FORBID
#define fopen FOPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID
...@@ -31,6 +32,8 @@ extern "C" { ...@@ -31,6 +32,8 @@ extern "C" {
#define fstat FSTAT_FUNC_TAOS_FORBID #define fstat FSTAT_FUNC_TAOS_FORBID
#define close CLOSE_FUNC_TAOS_FORBID #define close CLOSE_FUNC_TAOS_FORBID
#define fclose FCLOSE_FUNC_TAOS_FORBID #define fclose FCLOSE_FUNC_TAOS_FORBID
#define fsync FSYNC_FUNC_TAOS_FORBID
// #define fflush FFLUSH_FUNC_TAOS_FORBID
#endif #endif
#ifndef PATH_MAX #ifndef PATH_MAX
...@@ -47,13 +50,13 @@ typedef struct TdFile *TdFilePtr; ...@@ -47,13 +50,13 @@ typedef struct TdFile *TdFilePtr;
#define TD_FILE_TEXT 0x0020 #define TD_FILE_TEXT 0x0020
#define TD_FILE_AUTO_DEL 0x0040 #define TD_FILE_AUTO_DEL 0x0040
#define TD_FILE_EXCL 0x0080 #define TD_FILE_EXCL 0x0080
#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile #define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
#define TD_FILE_ACCESS_EXIST_OK 0x1 #define TD_FILE_ACCESS_EXIST_OK 0x1
#define TD_FILE_ACCESS_READ_OK 0x2 #define TD_FILE_ACCESS_READ_OK 0x2
#define TD_FILE_ACCESS_WRITE_OK 0x4 #define TD_FILE_ACCESS_WRITE_OK 0x4
bool taosCheckAccessFile(const char *pathname, int mode); bool taosCheckAccessFile(const char *pathname, int mode);
int32_t taosLockFile(TdFilePtr pFile); int32_t taosLockFile(TdFilePtr pFile);
int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUnLockFile(TdFilePtr pFile);
...@@ -80,10 +83,10 @@ int64_t taosCloseFile(TdFilePtr *ppFile); ...@@ -80,10 +83,10 @@ int64_t taosCloseFile(TdFilePtr *ppFile);
int32_t taosRenameFile(const char *oldName, const char *newName); int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(const char *from, const char *to); int64_t taosCopyFile(const char *from, const char *to);
int32_t taosRemoveFile(const char *path);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
......
...@@ -16,6 +16,14 @@ ...@@ -16,6 +16,14 @@
#ifndef _TD_OS_SOCKET_H_ #ifndef _TD_OS_SOCKET_H_
#define _TD_OS_SOCKET_H_ #define _TD_OS_SOCKET_H_
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define socket SOCKET_FUNC_TAOS_FORBID
#define bind BIND_FUNC_TAOS_FORBID
#define listen LISTEN_FUNC_TAOS_FORBID
// #define accept ACCEPT_FUNC_TAOS_FORBID
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#include "winsock2.h" #include "winsock2.h"
#include <WS2tcpip.h> #include <WS2tcpip.h>
...@@ -30,6 +38,8 @@ ...@@ -30,6 +38,8 @@
extern "C" { extern "C" {
#endif #endif
#ifndef USE_UV
#define TAOS_EPOLL_WAIT_TIME 500 #define TAOS_EPOLL_WAIT_TIME 500
typedef int32_t SOCKET; typedef int32_t SOCKET;
typedef SOCKET EpollFd; typedef SOCKET EpollFd;
...@@ -50,7 +60,6 @@ void taosShutDownSocketRD(SOCKET fd); ...@@ -50,7 +60,6 @@ void taosShutDownSocketRD(SOCKET fd);
void taosShutDownSocketWR(SOCKET fd); void taosShutDownSocketWR(SOCKET fd);
int32_t taosSetNonblocking(SOCKET sock, int32_t on); int32_t taosSetNonblocking(SOCKET sock, int32_t on);
void taosIgnSIGPIPE(); void taosIgnSIGPIPE();
void taosBlockSIGPIPE();
void taosSetMaskSIGPIPE(); void taosSetMaskSIGPIPE();
int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
...@@ -86,6 +95,10 @@ uint32_t taosGetIpv4FromFqdn(const char *); ...@@ -86,6 +95,10 @@ uint32_t taosGetIpv4FromFqdn(const char *);
void tinet_ntoa(char *ipstr, uint32_t ip); void tinet_ntoa(char *ipstr, uint32_t ip);
uint32_t ip2uint(const char *const ip_addr); uint32_t ip2uint(const char *const ip_addr);
#endif
void taosBlockSIGPIPE();
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
...@@ -38,15 +38,13 @@ int32_t taosGetEmail(char *email, int32_t maxLen); ...@@ -38,15 +38,13 @@ int32_t taosGetEmail(char *email, int32_t maxLen);
int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
int32_t taosGetCpuCores(float *numOfCores); int32_t taosGetCpuCores(float *numOfCores);
int32_t taosGetCpuUsage(float *cpu_system, float *cpu_engine); int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine);
int32_t taosGetTotalMemory(int64_t *totalKB); int32_t taosGetTotalMemory(int64_t *totalKB);
int32_t taosGetProcMemory(int64_t *usedKB); int32_t taosGetProcMemory(int64_t *usedKB);
int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetSysMemory(int64_t *usedKB);
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars); int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
int32_t taosGetProcIO(float *readKB, float *writeKB); int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
int32_t taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
int32_t taosGetBandSpeed(float *bandSpeedKb);
int32_t taosSystem(const char *cmd); int32_t taosSystem(const char *cmd);
void taosKillSystem(); void taosKillSystem();
......
...@@ -40,55 +40,9 @@ typedef struct SCacheStatis { ...@@ -40,55 +40,9 @@ typedef struct SCacheStatis {
int64_t refreshCount; int64_t refreshCount;
} SCacheStatis; } SCacheStatis;
struct STrashElem; typedef struct SCacheObj SCacheObj;
typedef struct SCacheIter SCacheIter;
typedef struct SCacheDataNode { typedef struct STrashElem STrashElem;
uint64_t addedTime; // the added time when this element is added or updated into cache
uint64_t lifespan; // life duration when this element should be remove from cache
uint64_t expireTime; // expire time
uint64_t signature;
struct STrashElem *pTNodeHeader; // point to trash node head
uint16_t keySize : 15; // max key size: 32kb
bool inTrashcan : 1; // denote if it is in trash or not
uint32_t size; // allocated size for current SCacheDataNode
T_REF_DECLARE()
char *key;
char data[];
} SCacheDataNode;
typedef struct STrashElem {
struct STrashElem *prev;
struct STrashElem *next;
SCacheDataNode *pData;
} STrashElem;
/*
* to accommodate the old data which has the same key value of new one in hashList
* when an new node is put into cache, if an existed one with the same key:
* 1. if the old one does not be referenced, update it.
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
typedef struct {
int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included.
int64_t refreshTime;
STrashElem *pTrash;
char *name;
SCacheStatis statistics;
SHashObj *pHashTable;
__cache_free_fn_t freeFp;
uint32_t numOfElemsInTrash; // number of element in trash
uint8_t deleting; // set the deleting flag to stop refreshing ASAP.
pthread_t refreshWorker;
bool extendLifespan; // auto extend life span when one item is accessed.
int64_t checkTick; // tick used to record the check times of the refresh threads
#if defined(LINUX)
pthread_rwlock_t lock;
#else
pthread_mutex_t lock;
#endif
} SCacheObj;
/** /**
* initialize the cache object * initialize the cache object
...@@ -141,7 +95,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data); ...@@ -141,7 +95,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data);
* @param data * @param data
* @return * @return
*/ */
void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); void *taosCacheTransferData(SCacheObj *pCacheObj, void **data);
/** /**
* remove data in cache, the data will not be removed immediately. * remove data in cache, the data will not be removed immediately.
...@@ -152,6 +106,13 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); ...@@ -152,6 +106,13 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data);
*/ */
void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove); void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
/**
*
* @param pCacheObj
* @return
*/
size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
/** /**
* move all data node into trash, clear node in trash can if it is not referenced by any clients * move all data node into trash, clear node in trash can if it is not referenced by any clients
* @param handle * @param handle
...@@ -184,6 +145,12 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1); ...@@ -184,6 +145,12 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
*/ */
void taosStopCacheRefreshWorker(); void taosStopCacheRefreshWorker();
SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
bool taosCacheIterNext(SCacheIter* pIter);
void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
void taosCacheDestroyIter(SCacheIter* pIter);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
...@@ -206,6 +206,7 @@ typedef enum ELogicConditionType { ...@@ -206,6 +206,7 @@ typedef enum ELogicConditionType {
#define TSDB_FUNC_TYPE_AGGREGATE 2 #define TSDB_FUNC_TYPE_AGGREGATE 2
#define TSDB_FUNC_MAX_RETRIEVE 1024 #define TSDB_FUNC_MAX_RETRIEVE 1024
#define TSDB_INDEX_NAME_LEN 32
#define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN
......
...@@ -28,12 +28,8 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); ...@@ -28,12 +28,8 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_before_fn_t)(void *);
typedef void (*_hash_free_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *);
#define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define HASH_KEY_ALREADY_EXISTS (-2)
#define HASH_DEFAULT_LOAD_FACTOR (0.75) #define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS)
#define HASH_INDEX(v, c) ((v) & ((c)-1))
#define HASH_NODE_EXIST(code) (code == -2)
/** /**
* murmur hash algorithm * murmur hash algorithm
...@@ -54,49 +50,15 @@ uint32_t taosIntHash_32(const char *key, uint32_t len); ...@@ -54,49 +50,15 @@ uint32_t taosIntHash_32(const char *key, uint32_t len);
uint32_t taosIntHash_64(const char *key, uint32_t len); uint32_t taosIntHash_64(const char *key, uint32_t len);
_hash_fn_t taosGetDefaultHashFunction(int32_t type); _hash_fn_t taosGetDefaultHashFunction(int32_t type);
_equal_fn_t taosGetDefaultEqualFunction(int32_t type); _equal_fn_t taosGetDefaultEqualFunction(int32_t type);
typedef struct SHashNode {
struct SHashNode *next;
uint32_t hashVal; // the hash value of key
uint32_t dataLen; // length of data
uint32_t keyLen; // length of the key
uint16_t count; // reference count
int8_t removed; // flag to indicate removed
char data[];
} SHashNode;
#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
typedef enum SHashLockTypeE { typedef enum SHashLockTypeE {
HASH_NO_LOCK = 0, HASH_NO_LOCK = 0,
HASH_ENTRY_LOCK = 1, HASH_ENTRY_LOCK = 1,
} SHashLockTypeE; } SHashLockTypeE;
typedef struct SHashEntry { typedef struct SHashNode SHashNode;
int32_t num; // number of elements in current entry typedef struct SHashObj SHashObj;
SRWLatch latch; // entry latch
SHashNode *next;
} SHashEntry;
typedef struct SHashObj {
SHashEntry **hashList;
uint32_t capacity; // number of slots
uint32_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function
_hash_free_fn_t freeFp; // hash node free callback function
_equal_fn_t equalFp; // equal function
_hash_before_fn_t callbackFp; // function invoked before return the value to caller
SRWLatch lock; // read-write spin lock
SHashLockTypeE type; // lock type
bool enableUpdate; // enable update
SArray *pMemBlock; // memory block allocated for SHashEntry
} SHashObj;
/** /**
* init the hash table * init the hash table
...@@ -126,8 +88,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj); ...@@ -126,8 +88,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj);
*/ */
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size);
int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded);
/** /**
* return the payload data with the specified key * return the payload data with the specified key
* *
...@@ -146,17 +106,18 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); ...@@ -146,17 +106,18 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
* @param destBuf * @param destBuf
* @return * @return
*/ */
void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf); int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf);
/** /**
* Clone the result to interval allocated buffer *
* @param pHashObj * @param pHashObj
* @param key * @param key
* @param keyLen * @param keyLen
* @param destBuf * @param destBuf
* @param size
* @return * @return
*/ */
void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz); int32_t taosHashGetDup_m(SHashObj* pHashObj, const void* key, size_t keyLen, void** destBuf, int32_t* size);
/** /**
* remove item with the specified key * remove item with the specified key
...@@ -207,37 +168,13 @@ void *taosHashIterate(SHashObj *pHashObj, void *p); ...@@ -207,37 +168,13 @@ void *taosHashIterate(SHashObj *pHashObj, void *p);
*/ */
void taosHashCancelIterate(SHashObj *pHashObj, void *p); void taosHashCancelIterate(SHashObj *pHashObj, void *p);
/** /**
* Get the corresponding key information for a given data in hash table * Get the corresponding key information for a given data in hash table
* @param data * @param data
* @return * @param keyLen
*/ * @return
int32_t taosHashGetKey(void *data, void **key, size_t *keyLen); */
void *taosHashGetKey(void *data, size_t* keyLen);
/**
* Get the corresponding key information for a given data in hash table, using memcpy
* @param data
* @param dst
* @return
*/
static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) {
if (NULL == data || NULL == dst) {
return -1;
}
SHashNode *node = GET_HASH_PNODE(data);
void *key = GET_HASH_NODE_KEY(node);
memcpy(dst, key, node->keyLen);
return 0;
}
/**
* Get the corresponding data length for a given data in hash table
* @param data
* @return
*/
int32_t taosHashGetDataLen(void *data);
/** /**
* return the payload data with the specified key(reference number added) * return the payload data with the specified key(reference number added)
...@@ -258,8 +195,20 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); ...@@ -258,8 +195,20 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen);
*/ */
void taosHashRelease(SHashObj *pHashObj, void *p); void taosHashRelease(SHashObj *pHashObj, void *p);
/**
*
* @param pHashObj
* @param fp
*/
void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
/**
*
* @param pHashObj
* @param fp
*/
void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
...@@ -22,9 +22,29 @@ ...@@ -22,9 +22,29 @@
extern "C" { extern "C" {
#endif #endif
typedef enum {
DEBUG_FATAL = 1,
DEBUG_ERROR = 1,
DEBUG_WARN = 2,
DEBUG_INFO = 2,
DEBUG_DEBUG = 4,
DEBUG_TRACE = 8,
DEBUG_DUMP = 16,
DEBUG_SCREEN = 64,
DEBUG_FILE = 128
} ELogLevel;
typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content);
extern bool tsLogEmbedded;
extern bool tsAsyncLog; extern bool tsAsyncLog;
extern int32_t tsNumOfLogLines; extern int32_t tsNumOfLogLines;
extern int32_t tsLogKeepDays; extern int32_t tsLogKeepDays;
extern LogFp tsLogFp;
extern int64_t tsNumOfErrorLogs;
extern int64_t tsNumOfInfoLogs;
extern int64_t tsNumOfDebugLogs;
extern int64_t tsNumOfTraceLogs;
extern int32_t dDebugFlag; extern int32_t dDebugFlag;
extern int32_t vDebugFlag; extern int32_t vDebugFlag;
extern int32_t mDebugFlag; extern int32_t mDebugFlag;
...@@ -40,45 +60,33 @@ extern int32_t tsdbDebugFlag; ...@@ -40,45 +60,33 @@ extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag; extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag; extern int32_t fsDebugFlag;
#define DEBUG_FATAL 1U
#define DEBUG_ERROR DEBUG_FATAL
#define DEBUG_WARN 2U
#define DEBUG_INFO DEBUG_WARN
#define DEBUG_DEBUG 4U
#define DEBUG_TRACE 8U
#define DEBUG_DUMP 16U
#define DEBUG_SCREEN 64U
#define DEBUG_FILE 128U
int32_t taosInitLog(const char *logName, int32_t maxFiles); int32_t taosInitLog(const char *logName, int32_t maxFiles);
void taosCloseLog(); void taosCloseLog();
void taosResetLog(); void taosResetLog();
void taosSetAllDebugFlag(int32_t flag); void taosSetAllDebugFlag(int32_t flag);
void taosDumpData(uint8_t *msg, int32_t len); void taosDumpData(uint8_t *msg, int32_t len);
void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...)
#ifdef __GNUC__ #ifdef __GNUC__
__attribute__((format(printf, 3, 4))) __attribute__((format(printf, 4, 5)))
#endif #endif
; ;
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...)
#ifdef __GNUC__ #ifdef __GNUC__
__attribute__((format(printf, 3, 4))) __attribute__((format(printf, 4, 5)))
#endif #endif
; ;
extern int8_t tscEmbeddedInUtil; #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} #define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} #define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", DEBUG_INFO, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }}
#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -53,7 +53,7 @@ typedef struct SDiskbasedBufStatis { ...@@ -53,7 +53,7 @@ typedef struct SDiskbasedBufStatis {
* @param handle * @param handle
* @return * @return
*/ */
int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir);
/** /**
* *
......
...@@ -20,12 +20,12 @@ ...@@ -20,12 +20,12 @@
extern "C" { extern "C" {
#endif #endif
#include "tcommon.h"
#include "parser.h" #include "parser.h"
#include "query.h" #include "query.h"
#include "taos.h" #include "taos.h"
#include "tcommon.h"
#include "tdatablock.h"
#include "tdef.h" #include "tdef.h"
#include "tep.h"
#include "thash.h" #include "thash.h"
#include "tlist.h" #include "tlist.h"
#include "tmsg.h" #include "tmsg.h"
...@@ -47,12 +47,12 @@ extern "C" { ...@@ -47,12 +47,12 @@ extern "C" {
typedef struct SAppInstInfo SAppInstInfo; typedef struct SAppInstInfo SAppInstInfo;
typedef struct SHbConnInfo { typedef struct {
void* param; void* param;
SClientHbReq* req; SClientHbReq* req;
} SHbConnInfo; } SHbConnInfo;
typedef struct SAppHbMgr { typedef struct {
char* key; char* key;
// statistics // statistics
int32_t reportCnt; int32_t reportCnt;
...@@ -68,11 +68,11 @@ typedef struct SAppHbMgr { ...@@ -68,11 +68,11 @@ typedef struct SAppHbMgr {
SHashObj* connInfo; // hash<SClientHbKey, SHbConnInfo> SHashObj* connInfo; // hash<SClientHbKey, SHbConnInfo>
} SAppHbMgr; } SAppHbMgr;
typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req);
typedef struct SClientHbMgr { typedef struct {
int8_t inited; int8_t inited;
// ctl // ctl
int8_t threadStop; int8_t threadStop;
...@@ -108,13 +108,13 @@ typedef struct SHeartBeatInfo { ...@@ -108,13 +108,13 @@ typedef struct SHeartBeatInfo {
} SHeartBeatInfo; } SHeartBeatInfo;
struct SAppInstInfo { struct SAppInstInfo {
int64_t numOfConns; int64_t numOfConns;
SCorEpSet mgmtEp; SCorEpSet mgmtEp;
SInstanceSummary summary; SInstanceSummary summary;
SList* pConnList; // STscObj linked list SList* pConnList; // STscObj linked list
int64_t clusterId; int64_t clusterId;
void* pTransporter; void* pTransporter;
struct SAppHbMgr* pAppHbMgr; SAppHbMgr* pAppHbMgr;
}; };
typedef struct SAppInfo { typedef struct SAppInfo {
...@@ -141,10 +141,6 @@ typedef struct STscObj { ...@@ -141,10 +141,6 @@ typedef struct STscObj {
SAppInstInfo* pAppInfo; SAppInstInfo* pAppInfo;
} STscObj; } STscObj;
typedef struct SMqConsumer {
STscObj* pTscObj;
} SMqConsumer;
typedef struct SReqResultInfo { typedef struct SReqResultInfo {
const char* pRspMsg; const char* pRspMsg;
const char* pData; const char* pData;
......
...@@ -22,13 +22,13 @@ extern "C" { ...@@ -22,13 +22,13 @@ extern "C" {
#include "tlog.h" #include "tlog.h"
#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -23,7 +23,7 @@ static SClientHbMgr clientHbMgr = {0}; ...@@ -23,7 +23,7 @@ static SClientHbMgr clientHbMgr = {0};
static int32_t hbCreateThread(); static int32_t hbCreateThread();
static void hbStopThread(); static void hbStopThread();
static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
int32_t code = 0; int32_t code = 0;
...@@ -104,7 +104,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo ...@@ -104,7 +104,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
if (NULL == info) { if (NULL == info) {
tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType); tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType);
...@@ -163,7 +163,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRs ...@@ -163,7 +163,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRs
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t hbMqAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) {
static int32_t emptyRspNum = 0; static int32_t emptyRspNum = 0;
if (code != 0) { if (code != 0) {
tfree(param); tfree(param);
...@@ -224,9 +224,14 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl ...@@ -224,9 +224,14 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
SDbVgVersion *db = &dbs[i]; SDbVgVersion *db = &dbs[i];
db->dbId = htobe64(db->dbId); db->dbId = htobe64(db->dbId);
db->vgVersion = htonl(db->vgVersion); db->vgVersion = htonl(db->vgVersion);
db->numOfTable = htonl(db->numOfTable);
} }
SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = sizeof(SDbVgVersion) * dbNum, .value = dbs}; SKv kv = {
.key = HEARTBEAT_KEY_DBINFO,
.valueLen = sizeof(SDbVgVersion) * dbNum,
.value = dbs,
};
tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen); tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen);
...@@ -256,7 +261,11 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC ...@@ -256,7 +261,11 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
stb->tversion = htons(stb->tversion); stb->tversion = htons(stb->tversion);
} }
SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs}; SKv kv = {
.key = HEARTBEAT_KEY_STBINFO,
.valueLen = sizeof(SSTableMetaVersion) * stbNum,
.value = stbs,
};
tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen);
...@@ -288,7 +297,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req ...@@ -288,7 +297,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {} int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; }
void hbMgrInitMqHbHandle() { void hbMgrInitMqHbHandle() {
clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle; clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle;
...@@ -396,7 +405,7 @@ static void *hbThreadFunc(void *param) { ...@@ -396,7 +405,7 @@ static void *hbThreadFunc(void *param) {
free(buf); free(buf);
break; break;
} }
pInfo->fp = hbMqAsyncCallBack; pInfo->fp = hbAsyncCallBack;
pInfo->msgInfo.pData = buf; pInfo->msgInfo.pData = buf;
pInfo->msgInfo.len = tlen; pInfo->msgInfo.len = tlen;
pInfo->msgType = TDMT_MND_HEARTBEAT; pInfo->msgType = TDMT_MND_HEARTBEAT;
...@@ -448,7 +457,6 @@ static void hbStopThread() { ...@@ -448,7 +457,6 @@ static void hbStopThread() {
} }
SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
/*return NULL;*/
hbMgrInit(); hbMgrInit();
SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr)); SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr));
if (pAppHbMgr == NULL) { if (pAppHbMgr == NULL) {
...@@ -473,7 +481,8 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { ...@@ -473,7 +481,8 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
free(pAppHbMgr); free(pAppHbMgr);
return NULL; return NULL;
} }
pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq;
taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq);
// init getInfoFunc // init getInfoFunc
pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
...@@ -506,7 +515,6 @@ void appHbMgrCleanup(void) { ...@@ -506,7 +515,6 @@ void appHbMgrCleanup(void) {
} }
int hbMgrInit() { int hbMgrInit() {
/*return 0;*/
// init once // init once
int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1);
if (old == 1) return 0; if (old == 1) return 0;
...@@ -524,7 +532,7 @@ int hbMgrInit() { ...@@ -524,7 +532,7 @@ int hbMgrInit() {
} }
void hbMgrCleanUp() { void hbMgrCleanUp() {
return; #if 0
hbStopThread(); hbStopThread();
// destroy all appHbMgr // destroy all appHbMgr
...@@ -537,6 +545,7 @@ void hbMgrCleanUp() { ...@@ -537,6 +545,7 @@ void hbMgrCleanUp() {
pthread_mutex_unlock(&clientHbMgr.lock); pthread_mutex_unlock(&clientHbMgr.lock);
clientHbMgr.appHbMgrs = NULL; clientHbMgr.appHbMgrs = NULL;
#endif
} }
int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) {
...@@ -563,9 +572,11 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo * ...@@ -563,9 +572,11 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *
} }
int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) {
/*return 0;*/ SClientHbKey connKey = {
SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; .connId = connId,
SHbConnInfo info = {0}; .hbType = HEARTBEAT_TYPE_QUERY,
};
SHbConnInfo info = {0};
switch (hbType) { switch (hbType) {
case HEARTBEAT_TYPE_QUERY: { case HEARTBEAT_TYPE_QUERY: {
...@@ -586,7 +597,6 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3 ...@@ -586,7 +597,6 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3
} }
void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
/*return;*/
int32_t code = 0; int32_t code = 0;
code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
...@@ -598,7 +608,6 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { ...@@ -598,7 +608,6 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen, int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen,
int32_t valueLen) { int32_t valueLen) {
return 0;
// find req by connection id // find req by connection id
SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
ASSERT(pReq != NULL); ASSERT(pReq != NULL);
......
...@@ -4,8 +4,8 @@ ...@@ -4,8 +4,8 @@
#include "parser.h" #include "parser.h"
#include "planner.h" #include "planner.h"
#include "scheduler.h" #include "scheduler.h"
#include "tdatablock.h"
#include "tdef.h" #include "tdef.h"
#include "tep.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsgtype.h" #include "tmsgtype.h"
#include "tpagedbuf.h" #include "tpagedbuf.h"
......
此差异已折叠。
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tep.h" #include "tdatablock.h"
#include "tcompare.h" #include "tcompare.h"
#include "tglobal.h" #include "tglobal.h"
......
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "tcompare.h" #include "tcompare.h"
#include "tconfig.h" #include "tconfig.h"
#include "tep.h" #include "tdatablock.h"
#include "tlog.h" #include "tlog.h"
SConfig *tsCfg = NULL; SConfig *tsCfg = NULL;
...@@ -155,21 +155,40 @@ static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t prima ...@@ -155,21 +155,40 @@ static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t prima
uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
} }
static void taosSetTfsCfg(SConfig *pCfg) { static int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
if (pItem == NULL) return; memset(tsDataDir, 0, PATH_MAX);
int32_t size = taosArrayGetSize(pItem->array); int32_t size = taosArrayGetSize(pItem->array);
if (size <= 0) { if (size <= 0) {
tsDiskCfgNum = 1; tsDiskCfgNum = 1;
taosAddDataDir(0, pItem->str, 0, 1); taosAddDataDir(0, pItem->str, 0, 1);
tstrncpy(tsDataDir, pItem->str, PATH_MAX);
if (taosMkDir(tsDataDir) != 0) {
uError("failed to create dataDir:%s since %s", tsDataDir, terrstr());
return -1;
}
} else { } else {
tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS; tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS;
for (int32_t index = 0; index < tsDiskCfgNum; ++index) { for (int32_t index = 0; index < tsDiskCfgNum; ++index) {
SDiskCfg *pCfg = taosArrayGet(pItem->array, index); SDiskCfg *pCfg = taosArrayGet(pItem->array, index);
memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg));
if (pCfg->level == 0 && pCfg->primary == 1) {
tstrncpy(tsDataDir, pCfg->dir, PATH_MAX);
}
if (taosMkDir(pCfg->dir) != 0) {
uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr());
return -1;
}
} }
} }
if (tsDataDir[0] == 0) {
uError("datadir not set");
return -1;
}
return 0;
} }
struct SConfig *taosGetCfg() { struct SConfig *taosGetCfg() {
...@@ -358,7 +377,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { ...@@ -358,7 +377,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
} }
static void taosSetClientCfg(SConfig *pCfg) { static int32_t taosSetClientCfg(SConfig *pCfg) {
tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN);
tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
...@@ -375,9 +394,13 @@ static void taosSetClientCfg(SConfig *pCfg) { ...@@ -375,9 +394,13 @@ static void taosSetClientCfg(SConfig *pCfg) {
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port);
cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype);
tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX);
taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); taosExpandDir(tsTempDir, tsTempDir, PATH_MAX);
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval;
if (taosMkDir(tsTempDir) != 0) {
uError("failed to create tempDir:%s since %s", tsTempDir, terrstr());
return -1;
}
tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval;
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
...@@ -392,6 +415,8 @@ static void taosSetClientCfg(SConfig *pCfg) { ...@@ -392,6 +415,8 @@ static void taosSetClientCfg(SConfig *pCfg) {
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
return 0;
} }
static void taosSetSystemCfg(SConfig *pCfg) { static void taosSetSystemCfg(SConfig *pCfg) {
...@@ -411,11 +436,8 @@ static void taosSetSystemCfg(SConfig *pCfg) { ...@@ -411,11 +436,8 @@ static void taosSetSystemCfg(SConfig *pCfg) {
tsVersion = 30000000; tsVersion = 30000000;
} }
static void taosSetServerCfg(SConfig *pCfg) { static int32_t taosSetServerCfg(SConfig *pCfg) {
tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
taosExpandDir(tsDataDir, tsDataDir, PATH_MAX);
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
...@@ -444,6 +466,8 @@ static void taosSetServerCfg(SConfig *pCfg) { ...@@ -444,6 +466,8 @@ static void taosSetServerCfg(SConfig *pCfg) {
if (tsQueryBufferSize >= 0) { if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
} }
return 0;
} }
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
...@@ -454,10 +478,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi ...@@ -454,10 +478,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
if (pCfg == NULL) return -1; if (pCfg == NULL) return -1;
if (tsc) { if (tsc) {
tscEmbeddedInUtil = 0; tsLogEmbedded = 0;
if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddClientLogCfg(pCfg) != 0) return -1;
} else { } else {
tscEmbeddedInUtil = 1; tsLogEmbedded = 1;
if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddClientLogCfg(pCfg) != 0) return -1;
if (taosAddServerLogCfg(pCfg) != 0) return -1; if (taosAddServerLogCfg(pCfg) != 0) return -1;
} }
...@@ -504,8 +528,8 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU ...@@ -504,8 +528,8 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
tsCfg = cfgInit(); tsCfg = cfgInit();
if (tsc) { if (tsc) {
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
if (taosAddClientCfg(tsCfg) != 0) return -1; if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
} else { } else {
if (taosAddClientCfg(tsCfg) != 0) return -1; if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddServerCfg(tsCfg) != 0) return -1; if (taosAddServerCfg(tsCfg) != 0) return -1;
...@@ -528,24 +552,14 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU ...@@ -528,24 +552,14 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
} }
if (tsc) { if (tsc) {
taosSetClientCfg(tsCfg); if (taosSetClientCfg(tsCfg)) return -1;
} else { } else {
taosSetClientCfg(tsCfg); if (taosSetClientCfg(tsCfg)) return -1;
taosSetServerCfg(tsCfg); if (taosSetServerCfg(tsCfg)) return -1;
taosSetTfsCfg(tsCfg); if (taosSetTfsCfg(tsCfg) != 0) return -1;
} }
taosSetSystemCfg(tsCfg); taosSetSystemCfg(tsCfg);
if (taosMkDir(tsTempDir) != 0) {
uError("failed to create dir:%s since %s", tsTempDir, terrstr());
return -1;
}
if (!tsc && taosMkDir(tsDataDir) != 0) {
uError("failed to create dir:%s since %s", tsDataDir, terrstr());
return -1;
}
cfgDumpCfg(tsCfg, tsc, false); cfgDumpCfg(tsCfg, tsc, false);
return 0; return 0;
} }
......
...@@ -1423,6 +1423,7 @@ int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { ...@@ -1423,6 +1423,7 @@ int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) {
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbId) < 0) return -1; if (tEncodeI64(&encoder, pReq->dbId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfTable) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
...@@ -1438,6 +1439,7 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { ...@@ -1438,6 +1439,7 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) {
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbId) < 0) return -1; if (tDecodeI64(&decoder, &pReq->dbId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfTable) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);
tCoderClear(&decoder); tCoderClear(&decoder);
...@@ -1482,6 +1484,7 @@ static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { ...@@ -1482,6 +1484,7 @@ static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) {
if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1; if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1;
if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1; if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1;
if (tEncodeI32(pEncoder, pVgInfo->numOfTable) < 0) return -1;
} }
return 0; return 0;
...@@ -1542,6 +1545,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { ...@@ -1542,6 +1545,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1; if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1;
if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1; if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1; if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1;
if (tDecodeI32(pDecoder, &vgInfo.numOfTable) < 0) return -1;
taosArrayPush(pRsp->pVgroupInfos, &vgInfo); taosArrayPush(pRsp->pVgroupInfos, &vgInfo);
} }
......
...@@ -46,7 +46,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { ...@@ -46,7 +46,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
} }
if (!autoDelete) { if (!autoDelete) {
remove(pTSBuf->path); taosRemoveFile(pTSBuf->path);
} }
if (NULL == allocResForTSBuf(pTSBuf)) { if (NULL == allocResForTSBuf(pTSBuf)) {
...@@ -178,7 +178,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { ...@@ -178,7 +178,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
if (pTSBuf->autoDelete) { if (pTSBuf->autoDelete) {
// ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path);
remove(pTSBuf->path); taosRemoveFile(pTSBuf->path);
} else { } else {
// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path);
} }
......
...@@ -8,7 +8,8 @@ ...@@ -8,7 +8,8 @@
#pragma GCC diagnostic ignored "-Wsign-compare" #pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h" #include "os.h"
#include "tep.h" #include "tcommon.h"
#include "tdatablock.h"
#include "tcommon.h" #include "tcommon.h"
#include "taos.h" #include "taos.h"
#include "tvariant.h" #include "tvariant.h"
......
...@@ -105,9 +105,19 @@ typedef struct { ...@@ -105,9 +105,19 @@ typedef struct {
} SBnodeMgmt; } SBnodeMgmt;
typedef struct { typedef struct {
int32_t openVnodes;
int32_t totalVnodes;
int32_t masterNum;
int64_t numOfSelectReqs;
int64_t numOfInsertReqs;
int64_t numOfInsertSuccessReqs;
int64_t numOfBatchInsertReqs;
int64_t numOfBatchInsertSuccessReqs;
} SVnodesStat;
typedef struct {
SVnodesStat stat;
SHashObj *hash; SHashObj *hash;
int32_t openVnodes;
int32_t totalVnodes;
SRWLatch latch; SRWLatch latch;
SQWorkerPool queryPool; SQWorkerPool queryPool;
SFWorkerPool fetchPool; SFWorkerPool fetchPool;
......
...@@ -23,9 +23,11 @@ extern "C" { ...@@ -23,9 +23,11 @@ extern "C" {
#include "os.h" #include "os.h"
#include "cJSON.h" #include "cJSON.h"
#include "monitor.h"
#include "tcache.h" #include "tcache.h"
#include "tcrc32c.h" #include "tcrc32c.h"
#include "tep.h" #include "tdatablock.h"
#include "tglobal.h"
#include "thash.h" #include "thash.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "tlog.h" #include "tlog.h"
...@@ -35,8 +37,6 @@ extern "C" { ...@@ -35,8 +37,6 @@ extern "C" {
#include "tthread.h" #include "tthread.h"
#include "ttime.h" #include "ttime.h"
#include "tworker.h" #include "tworker.h"
#include "tglobal.h"
#include "monitor.h"
#include "dnode.h" #include "dnode.h"
...@@ -47,14 +47,12 @@ extern "C" { ...@@ -47,14 +47,12 @@ extern "C" {
#include "vnode.h" #include "vnode.h"
#include "tfs.h" #include "tfs.h"
extern int32_t dDebugFlag; #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} #define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} #define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }} #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", 255, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;
......
...@@ -34,7 +34,6 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); ...@@ -34,7 +34,6 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo); SMonGrantInfo *pGrantInfo);
int8_t dndIsMnode(SDnode *pDnode);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -487,24 +487,21 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { ...@@ -487,24 +487,21 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
taosGetSysMemory(&pInfo->mem_system); taosGetSysMemory(&pInfo->mem_system);
pInfo->mem_total = tsTotalMemoryKB; pInfo->mem_total = tsTotalMemoryKB;
pInfo->disk_engine = 0; pInfo->disk_engine = 0;
pInfo->disk_used = tsDataSpace.size.used / (1024 * 1024 * 1024.0); pInfo->disk_used = tsDataSpace.size.used;
pInfo->disk_total = tsDataSpace.size.avail / (1024 * 1024 * 1024.0); pInfo->disk_total = tsDataSpace.size.total;
taosGetCardInfo(NULL, &pInfo->net_in, &pInfo->net_out); taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
taosGetProcIO(&pInfo->io_read, &pInfo->io_write); taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
pInfo->io_read_disk = 0;
pInfo->io_write_disk = 0; SVnodesStat *pStat = &pDnode->vmgmt.stat;
pInfo->req_select = 0; pInfo->req_select = pStat->numOfSelectReqs;
pInfo->req_select_rate = 0; pInfo->req_insert = pStat->numOfInsertReqs;
pInfo->req_insert = 0; pInfo->req_insert_success = pStat->numOfInsertSuccessReqs;
pInfo->req_insert_success = 0; pInfo->req_insert_batch = pStat->numOfBatchInsertReqs;
pInfo->req_insert_rate = 0; pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs;
pInfo->req_insert_batch = 0; pInfo->errors = tsNumOfErrorLogs;
pInfo->req_insert_batch_success = 0; pInfo->vnodes_num = pStat->totalVnodes;
pInfo->req_insert_batch_rate = 0; pInfo->masters = pStat->masterNum;
pInfo->errors = 0; pInfo->has_mnode = pDnode->mmgmt.deployed;
pInfo->vnodes_num = 0;
pInfo->masters = 0;
pInfo->has_mnode = dndIsMnode(pDnode);
} }
static void dndSendMonitorReport(SDnode *pDnode) { static void dndSendMonitorReport(SDnode *pDnode) {
......
...@@ -640,10 +640,3 @@ int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SM ...@@ -640,10 +640,3 @@ int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SM
dndReleaseMnode(pDnode, pMnode); dndReleaseMnode(pDnode, pMnode);
return code; return code;
} }
int8_t dndIsMnode(SDnode *pDnode) {
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL) return 0;
dndReleaseMnode(pDnode, pMnode);
return 1;
}
\ No newline at end of file
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
#include "dndVnodes.h" #include "dndVnodes.h"
#include "dndMgmt.h" #include "dndMgmt.h"
#include "dndTransport.h" #include "dndTransport.h"
#include "sync.h"
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
...@@ -85,7 +86,7 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { ...@@ -85,7 +86,7 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
int32_t refCount = 0; int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch); taosRLockLatch(&pMgmt->latch);
taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
if (pVnode == NULL) { if (pVnode == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
} else { } else {
...@@ -381,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) { ...@@ -381,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
char stepDesc[TSDB_STEP_DESC_LEN] = {0}; char stepDesc[TSDB_STEP_DESC_LEN] = {0};
snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId,
pMgmt->openVnodes, pMgmt->totalVnodes); pMgmt->stat.openVnodes, pMgmt->stat.totalVnodes);
dndReportStartup(pDnode, "open-vnodes", stepDesc); dndReportStartup(pDnode, "open-vnodes", stepDesc);
SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
...@@ -395,7 +396,7 @@ static void *dnodeOpenVnodeFunc(void *param) { ...@@ -395,7 +396,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
pThread->opened++; pThread->opened++;
} }
atomic_add_fetch_32(&pMgmt->openVnodes, 1); atomic_add_fetch_32(&pMgmt->stat.openVnodes, 1);
} }
dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
...@@ -421,7 +422,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { ...@@ -421,7 +422,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
return -1; return -1;
} }
pMgmt->totalVnodes = numOfVnodes; pMgmt->stat.totalVnodes = numOfVnodes;
int32_t threadNum = tsNumOfCores; int32_t threadNum = tsNumOfCores;
#if 1 #if 1
...@@ -469,11 +470,11 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { ...@@ -469,11 +470,11 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
free(threads); free(threads);
free(pCfgs); free(pCfgs);
if (pMgmt->openVnodes != pMgmt->totalVnodes) { if (pMgmt->stat.openVnodes != pMgmt->stat.totalVnodes) {
dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes); dError("there are total vnodes:%d, opened:%d", pMgmt->stat.totalVnodes, pMgmt->stat.openVnodes);
return -1; return -1;
} else { } else {
dInfo("total vnodes:%d open successfully", pMgmt->totalVnodes); dInfo("total vnodes:%d open successfully", pMgmt->stat.totalVnodes);
return 0; return 0;
} }
} }
...@@ -979,11 +980,18 @@ void dndCleanupVnodes(SDnode *pDnode) { ...@@ -979,11 +980,18 @@ void dndCleanupVnodes(SDnode *pDnode) {
void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) { void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt; SVnodesMgmt *pMgmt = &pDnode->vmgmt;
SVnodesStat *pStat = &pMgmt->stat;
int32_t totalVnodes = 0;
int32_t masterNum = 0;
int64_t numOfSelectReqs = 0;
int64_t numOfInsertReqs = 0;
int64_t numOfInsertSuccessReqs = 0;
int64_t numOfBatchInsertReqs = 0;
int64_t numOfBatchInsertSuccessReqs = 0;
taosRLockLatch(&pMgmt->latch); taosRLockLatch(&pMgmt->latch);
int32_t v = 0; void *pIter = taosHashIterate(pMgmt->hash, NULL);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) { while (pIter) {
SVnodeObj **ppVnode = pIter; SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue; if (ppVnode == NULL || *ppVnode == NULL) continue;
...@@ -993,8 +1001,24 @@ void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) { ...@@ -993,8 +1001,24 @@ void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
vnodeGetLoad(pVnode->pImpl, &vload); vnodeGetLoad(pVnode->pImpl, &vload);
taosArrayPush(pLoads, &vload); taosArrayPush(pLoads, &vload);
numOfSelectReqs += vload.numOfSelectReqs;
numOfInsertReqs += vload.numOfInsertReqs;
numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs;
numOfBatchInsertReqs += vload.numOfBatchInsertReqs;
numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs;
totalVnodes++;
if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++;
pIter = taosHashIterate(pMgmt->hash, pIter); pIter = taosHashIterate(pMgmt->hash, pIter);
} }
taosRUnLockLatch(&pMgmt->latch); taosRUnLockLatch(&pMgmt->latch);
pStat->totalVnodes = totalVnodes;
pStat->masterNum = masterNum;
pStat->numOfSelectReqs = numOfSelectReqs;
pStat->numOfInsertReqs = numOfInsertReqs;
pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs;
pStat->numOfBatchInsertReqs = numOfBatchInsertReqs;
pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
} }
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
*/ */
#include "sut.h" #include "sut.h"
#include "tep.h" #include "tdatablock.h"
static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
TestClient* client = (TestClient*)parent; TestClient* client = (TestClient*)parent;
......
...@@ -28,7 +28,7 @@ void Testbase::InitLog(const char* path) { ...@@ -28,7 +28,7 @@ void Testbase::InitLog(const char* path) {
wDebugFlag = 0; wDebugFlag = 0;
sDebugFlag = 0; sDebugFlag = 0;
tsdbDebugFlag = 0; tsdbDebugFlag = 0;
tscEmbeddedInUtil = 1; tsLogEmbedded = 1;
tsAsyncLog = 0; tsAsyncLog = 0;
taosRemoveDir(path); taosRemoveDir(path);
......
...@@ -34,46 +34,6 @@ ...@@ -34,46 +34,6 @@
extern "C" { extern "C" {
#endif #endif
extern int32_t mDebugFlag;
// mnode log function
#define mFatal(...) \
{ \
if (mDebugFlag & DEBUG_FATAL) { \
taosPrintLog("MND FATAL ", 255, __VA_ARGS__); \
} \
}
#define mError(...) \
{ \
if (mDebugFlag & DEBUG_ERROR) { \
taosPrintLog("MND ERROR ", 255, __VA_ARGS__); \
} \
}
#define mWarn(...) \
{ \
if (mDebugFlag & DEBUG_WARN) { \
taosPrintLog("MND WARN ", 255, __VA_ARGS__); \
} \
}
#define mInfo(...) \
{ \
if (mDebugFlag & DEBUG_INFO) { \
taosPrintLog("MND ", 255, __VA_ARGS__); \
} \
}
#define mDebug(...) \
{ \
if (mDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
} \
}
#define mTrace(...) \
{ \
if (mDebugFlag & DEBUG_TRACE) { \
taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
} \
}
typedef enum { typedef enum {
MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_START = 0,
MND_AUTH_ACCT_USER, MND_AUTH_ACCT_USER,
......
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#include "sdb.h" #include "sdb.h"
#include "tcache.h" #include "tcache.h"
#include "tep.h" #include "tdatablock.h"
#include "tglobal.h" #include "tglobal.h"
#include "tqueue.h" #include "tqueue.h"
#include "ttime.h" #include "ttime.h"
...@@ -31,6 +31,13 @@ ...@@ -31,6 +31,13 @@
extern "C" { extern "C" {
#endif #endif
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg);
typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode);
...@@ -85,6 +92,11 @@ typedef struct { ...@@ -85,6 +92,11 @@ typedef struct {
ESyncState state; ESyncState state;
} SSyncMgmt; } SSyncMgmt;
typedef struct {
int64_t expireTimeMS;
int64_t timeseriesAllowed;
} SGrantInfo;
typedef struct SMnode { typedef struct SMnode {
int32_t dnodeId; int32_t dnodeId;
int64_t clusterId; int64_t clusterId;
...@@ -105,6 +117,7 @@ typedef struct SMnode { ...@@ -105,6 +117,7 @@ typedef struct SMnode {
STelemMgmt telemMgmt; STelemMgmt telemMgmt;
SSyncMgmt syncMgmt; SSyncMgmt syncMgmt;
SHashObj *infosMeta; SHashObj *infosMeta;
SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX]; MndMsgFp msgFp[TDMT_MAX];
SendReqToDnodeFp sendReqToDnodeFp; SendReqToDnodeFp sendReqToDnodeFp;
SendReqToMnodeFp sendReqToMnodeFp; SendReqToMnodeFp sendReqToMnodeFp;
...@@ -120,7 +133,7 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); ...@@ -120,7 +133,7 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
uint64_t mndGenerateUid(char *name, int32_t len); uint64_t mndGenerateUid(char *name, int32_t len);
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -24,6 +24,7 @@ extern "C" { ...@@ -24,6 +24,7 @@ extern "C" {
int32_t mndInitProfile(SMnode *pMnode); int32_t mndInitProfile(SMnode *pMnode);
void mndCleanupProfile(SMnode *pMnode); void mndCleanupProfile(SMnode *pMnode);
int32_t mndGetNumOfConnections(SMnode *pMnode);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -885,6 +885,29 @@ DROP_DB_OVER: ...@@ -885,6 +885,29 @@ DROP_DB_OVER:
return code; return code;
} }
void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) {
int32_t vindex = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (vindex < pDb->cfg.numOfVgroups) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid == pDb->uid) {
*num += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT;
vindex++;
}
sdbRelease(pSdb, pVgroup);
}
sdbCancelFetch(pSdb, pIter);
}
static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
int32_t vindex = 0; int32_t vindex = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
...@@ -900,6 +923,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { ...@@ -900,6 +923,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
vgInfo.vgId = pVgroup->vgId; vgInfo.vgId = pVgroup->vgId;
vgInfo.hashBegin = pVgroup->hashBegin; vgInfo.hashBegin = pVgroup->hashBegin;
vgInfo.hashEnd = pVgroup->hashEnd; vgInfo.hashEnd = pVgroup->hashEnd;
vgInfo.numOfTable = pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT;
vgInfo.epset.numOfEps = pVgroup->replica; vgInfo.epset.numOfEps = pVgroup->replica;
for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { for (int32_t gid = 0; gid < pVgroup->replica; ++gid) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; SVnodeGid *pVgid = &pVgroup->vnodeGid[gid];
...@@ -967,7 +991,10 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { ...@@ -967,7 +991,10 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) {
goto USE_DB_OVER; goto USE_DB_OVER;
} }
if (usedbReq.vgVersion < pDb->vgVersion || usedbReq.dbId != pDb->uid) { int32_t numOfTable = 0;
mndGetDBTableNum(pDb, pMnode, &numOfTable);
if (usedbReq.vgVersion < pDb->vgVersion || usedbReq.dbId != pDb->uid || numOfTable != usedbReq.numOfTable) {
mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos);
} }
...@@ -1017,6 +1044,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, ...@@ -1017,6 +1044,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
SDbVgVersion *pDbVgVersion = &pDbs[i]; SDbVgVersion *pDbVgVersion = &pDbs[i];
pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId);
pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion);
pDbVgVersion->numOfTable = htonl(pDbVgVersion->numOfTable);
SUseDbRsp usedbRsp = {0}; SUseDbRsp usedbRsp = {0};
...@@ -1027,28 +1055,34 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, ...@@ -1027,28 +1055,34 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
usedbRsp.uid = pDbVgVersion->dbId; usedbRsp.uid = pDbVgVersion->dbId;
usedbRsp.vgVersion = -1; usedbRsp.vgVersion = -1;
taosArrayPush(batchUseRsp.pArray, &usedbRsp); taosArrayPush(batchUseRsp.pArray, &usedbRsp);
} else if (pDbVgVersion->vgVersion >= pDb->vgVersion) {
mDebug("db:%s, version not changed", pDbVgVersion->dbFName);
mndReleaseDb(pMnode, pDb);
continue; continue;
} else { }
usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
if (usedbRsp.pVgroupInfos == NULL) {
mndReleaseDb(pMnode, pDb);
mError("db:%s, failed to malloc usedb response", pDb->name);
continue;
}
mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); int32_t numOfTable = 0;
memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); mndGetDBTableNum(pDb, pMnode, &numOfTable);
usedbRsp.uid = pDb->uid;
usedbRsp.vgVersion = pDb->vgVersion;
usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos);
usedbRsp.hashMethod = pDb->hashMethod;
taosArrayPush(batchUseRsp.pArray, &usedbRsp); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) {
mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName);
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
continue;
} }
usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
if (usedbRsp.pVgroupInfos == NULL) {
mndReleaseDb(pMnode, pDb);
mError("db:%s, failed to malloc usedb response", pDb->name);
continue;
}
mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos);
memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN);
usedbRsp.uid = pDb->uid;
usedbRsp.vgVersion = pDb->vgVersion;
usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos);
usedbRsp.hashMethod = pDb->hashMethod;
taosArrayPush(batchUseRsp.pArray, &usedbRsp);
mndReleaseDb(pMnode, pDb);
} }
int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp);
......
...@@ -99,11 +99,15 @@ void mndUpdateMnodeRole(SMnode *pMnode) { ...@@ -99,11 +99,15 @@ void mndUpdateMnodeRole(SMnode *pMnode) {
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break; if (pIter == NULL) break;
ESyncState lastRole = pObj->role;
if (pObj->id == 1) { if (pObj->id == 1) {
pObj->role = TAOS_SYNC_STATE_LEADER; pObj->role = TAOS_SYNC_STATE_LEADER;
} else { } else {
pObj->role = TAOS_SYNC_STATE_CANDIDATE; pObj->role = TAOS_SYNC_STATE_CANDIDATE;
} }
if (pObj->role != lastRole) {
pObj->roleTime = taosGetTimestampMs();
}
sdbRelease(pSdb, pObj); sdbRelease(pSdb, pObj);
} }
......
...@@ -48,7 +48,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, ...@@ -48,7 +48,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
static void mndFreeConn(SConnObj *pConn); static void mndFreeConn(SConnObj *pConn);
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter);
static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter);
static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq);
static int32_t mndProcessConnectReq(SMnodeMsg *pReq); static int32_t mndProcessConnectReq(SMnodeMsg *pReq);
...@@ -158,27 +158,23 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { ...@@ -158,27 +158,23 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) {
taosCacheRelease(pMgmt->cache, (void **)&pConn, false); taosCacheRelease(pMgmt->cache, (void **)&pConn, false);
} }
static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) { void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SConnObj* pConn = NULL;
bool hasNext = taosCacheIterNext(pIter);
*pConn = NULL; if (hasNext) {
size_t dataLen = 0;
pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter); pConn = taosCacheIterGetData(pIter, &dataLen);
if (pIter == NULL) return NULL; } else {
taosCacheDestroyIter(pIter);
SCacheDataNode **pNode = pIter;
if (pNode == NULL || *pNode == NULL) {
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
return NULL;
} }
*pConn = (SConnObj *)((*pNode)->data); return pConn;
return pIter;
} }
static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; if (pIter != NULL) {
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); taosCacheDestroyIter(pIter);
}
} }
static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
...@@ -376,8 +372,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { ...@@ -376,8 +372,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
int32_t rspLen = 0; int32_t rspLen = 0;
mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) { if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv); taosArrayPush(hbRsp.info, &kv1);
} }
break; break;
} }
...@@ -386,8 +382,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { ...@@ -386,8 +382,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
int32_t rspLen = 0; int32_t rspLen = 0;
mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) { if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv); taosArrayPush(hbRsp.info, &kv1);
} }
break; break;
} }
...@@ -638,7 +634,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * ...@@ -638,7 +634,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
} }
pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); pShow->numOfRows = taosCacheGetNumOfObj(pMgmt->cache);
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbName, mndShowStr(pShow->type)); strcpy(pMeta->tbName, mndShowStr(pShow->type));
...@@ -653,8 +649,13 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in ...@@ -653,8 +649,13 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in
char *pWrite; char *pWrite;
char ipStr[TSDB_IPv4ADDR_LEN + 6]; char ipStr[TSDB_IPv4ADDR_LEN + 6];
if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->cache);
}
while (numOfRows < rows) { while (numOfRows < rows) {
pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); pConn = mndGetNextConn(pMnode, pShow->pIter);
if (pConn == NULL) break; if (pConn == NULL) break;
cols = 0; cols = 0;
...@@ -823,19 +824,24 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, ...@@ -823,19 +824,24 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
void *pIter; void *pIter;
char str[TSDB_IPv4ADDR_LEN + 6] = {0}; char str[TSDB_IPv4ADDR_LEN + 6] = {0};
if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->cache);
}
while (numOfRows < rows) { while (numOfRows < rows) {
pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); pConn = mndGetNextConn(pMnode, pShow->pIter);
if (pConn == NULL) { if (pConn == NULL) {
pShow->pIter = pIter; pShow->pIter = NULL;
break; break;
} }
if (numOfRows + pConn->numOfQueries >= rows) { if (numOfRows + pConn->numOfQueries >= rows) {
mndCancelGetNextConn(pMnode, pIter); taosCacheDestroyIter(pShow->pIter);
pShow->pIter = NULL;
break; break;
} }
pShow->pIter = pIter;
for (int32_t i = 0; i < pConn->numOfQueries; ++i) { for (int32_t i = 0; i < pConn->numOfQueries; ++i) {
SQueryDesc *pDesc = pConn->pQueries + i; SQueryDesc *pDesc = pConn->pQueries + i;
cols = 0; cols = 0;
...@@ -913,6 +919,12 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, ...@@ -913,6 +919,12 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
} }
static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; if (pIter != NULL) {
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); taosCacheDestroyIter(pIter);
}
} }
int32_t mndGetNumOfConnections(SMnode *pMnode) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
return taosCacheGetNumOfObj(pMgmt->cache);
}
\ No newline at end of file
...@@ -18,15 +18,15 @@ ...@@ -18,15 +18,15 @@
#include "mndCluster.h" #include "mndCluster.h"
#include "mndSync.h" #include "mndSync.h"
#include "tbuffer.h" #include "tbuffer.h"
#include "tjson.h"
#include "thttp.h" #include "thttp.h"
#include "tjson.h"
#define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_SERVER "telemetry.taosdata.com"
#define TELEMETRY_PORT 80 #define TELEMETRY_PORT 80
static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) { static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) {
SMnodeLoad load = {0}; SMnodeLoad load = {0};
if (mndGetLoad(pMnode, &load) != 0) return; mndGetLoad(pMnode, &load);
tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode); tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode);
tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode); tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode);
......
...@@ -359,6 +359,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { ...@@ -359,6 +359,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
return NULL; return NULL;
} }
mndUpdateMnodeRole(pMnode);
mDebug("mnode open successfully "); mDebug("mnode open successfully ");
return pMnode; return pMnode;
} }
...@@ -385,26 +386,6 @@ void mndDestroy(const char *path) { ...@@ -385,26 +386,6 @@ void mndDestroy(const char *path) {
mDebug("mnode is destroyed"); mDebug("mnode is destroyed");
} }
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
pLoad->numOfDnode = 0;
pLoad->numOfMnode = 0;
pLoad->numOfVgroup = 0;
pLoad->numOfDatabase = 0;
pLoad->numOfSuperTable = 0;
pLoad->numOfChildTable = 0;
pLoad->numOfColumn = 0;
pLoad->totalPoints = 0;
pLoad->totalStorage = 0;
pLoad->compStorage = 0;
return 0;
}
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
return 0;
}
SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg));
if (pMsg == NULL) { if (pMsg == NULL) {
...@@ -523,3 +504,131 @@ uint64_t mndGenerateUid(char *name, int32_t len) { ...@@ -523,3 +504,131 @@ uint64_t mndGenerateUid(char *name, int32_t len) {
} }
} while (true); } while (true);
} }
void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
memset(pLoad, 0, sizeof(SMnodeLoad));
SSdb *pSdb = pMnode->pSdb;
pLoad->numOfDnode = sdbGetSize(pSdb, SDB_DNODE);
pLoad->numOfMnode = sdbGetSize(pSdb, SDB_MNODE);
pLoad->numOfVgroup = sdbGetSize(pSdb, SDB_VGROUP);
pLoad->numOfDatabase = sdbGetSize(pSdb, SDB_DB);
pLoad->numOfSuperTable = sdbGetSize(pSdb, SDB_STB);
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
pLoad->numOfChildTable += pVgroup->numOfTables;
pLoad->numOfColumn += pVgroup->numOfTimeSeries;
pLoad->totalPoints += pVgroup->pointsWritten;
pLoad->totalStorage += pVgroup->totalStorage;
pLoad->compStorage += pVgroup->compStorage;
sdbRelease(pSdb, pVgroup);
}
}
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
if (!mndIsMaster(pMnode)) return -1;
SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs();
pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc));
pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc));
pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) {
return -1;
}
// cluster info
tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version));
pClusterInfo->monitor_interval = tsMonitorInterval;
pClusterInfo->connections_total = mndGetNumOfConnections(pMnode);
void *pIter = NULL;
while (1) {
SDnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SMonDnodeDesc desc = {0};
desc.dnode_id = pObj->id;
tstrncpy(desc.dnode_ep, pObj->ep, sizeof(desc.dnode_ep));
if (mndIsDnodeOnline(pMnode, pObj, ms)) {
tstrncpy(desc.status, "ready", sizeof(desc.status));
} else {
tstrncpy(desc.status, "offline", sizeof(desc.status));
}
taosArrayPush(pClusterInfo->dnodes, &desc);
sdbRelease(pSdb, pObj);
}
pIter = NULL;
while (1) {
SMnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SMonMnodeDesc desc = {0};
desc.mnode_id = pObj->id;
tstrncpy(desc.mnode_ep, pObj->pDnode->ep, sizeof(desc.mnode_ep));
tstrncpy(desc.role, mndGetRoleStr(pObj->role), sizeof(desc.role));
taosArrayPush(pClusterInfo->mnodes, &desc);
sdbRelease(pSdb, pObj);
if (pObj->role == TAOS_SYNC_STATE_LEADER) {
pClusterInfo->first_ep_dnode_id = pObj->id;
tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
pClusterInfo->master_uptime = (ms - pObj->roleTime) / (86400000.0f);
}
}
// vgroup info
pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
pClusterInfo->vgroups_total++;
SMonVgroupDesc desc = {0};
desc.vgroup_id = pVgroup->vgId;
strncpy(desc.database_name, pVgroup->dbName, sizeof(desc.database_name));
desc.tables_num = pVgroup->numOfTables;
pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries;
tstrncpy(desc.status, "unsynced", sizeof(desc.status));
for (int32_t i = 0; i < pVgroup->replica; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
SMonVnodeDesc *pVnDesc = &desc.vnodes[i];
pVnDesc->dnode_id = pVgid->dnodeId;
tstrncpy(pVnDesc->vnode_role, mndGetRoleStr(pVgid->role), sizeof(pVnDesc->vnode_role));
if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
tstrncpy(desc.status, "ready", sizeof(desc.status));
pClusterInfo->vgroups_alive++;
}
if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) {
pClusterInfo->vnodes_alive++;
}
pClusterInfo->vnodes_total++;
}
taosArrayPush(pVgroupInfo->vgroups, &desc);
sdbRelease(pSdb, pVgroup);
}
// grant info
pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f;
pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;
if (pMnode->grant.expireTimeMS == 0) {
pGrantInfo->expire_time = INT32_MAX;
pGrantInfo->timeseries_total = INT32_MAX;
}
return 0;
}
\ No newline at end of file
...@@ -28,12 +28,12 @@ ...@@ -28,12 +28,12 @@
extern "C" { extern "C" {
#endif #endif
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }} #define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }} #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }} #define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", 255, __VA_ARGS__); }} #define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
#define SDB_MAX_SIZE (32 * 1024) #define SDB_MAX_SIZE (32 * 1024)
......
...@@ -54,5 +54,5 @@ elseif(${META_DB_IMPL} STREQUAL "TDB") ...@@ -54,5 +54,5 @@ elseif(${META_DB_IMPL} STREQUAL "TDB")
endif() endif()
if(${BUILD_TEST}) if(${BUILD_TEST})
# add_subdirectory(test) add_subdirectory(test)
endif(${BUILD_TEST}) endif(${BUILD_TEST})
...@@ -24,8 +24,8 @@ ...@@ -24,8 +24,8 @@
extern "C" { extern "C" {
#endif #endif
#define META_SUPER_TABLE TD_SUPER_TABLE #define META_SUPER_TABLE TD_SUPER_TABLE
#define META_CHILD_TABLE TD_CHILD_TABLE #define META_CHILD_TABLE TD_CHILD_TABLE
#define META_NORMAL_TABLE TD_NORMAL_TABLE #define META_NORMAL_TABLE TD_NORMAL_TABLE
// Types exported // Types exported
...@@ -38,8 +38,10 @@ typedef struct SMetaCfg { ...@@ -38,8 +38,10 @@ typedef struct SMetaCfg {
typedef struct SMTbCursor SMTbCursor; typedef struct SMTbCursor SMTbCursor;
typedef struct SMCtbCursor SMCtbCursor; typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMSmaCursor SMSmaCursor;
typedef SVCreateTbReq STbCfg; typedef SVCreateTbReq STbCfg;
typedef STSma SSmaCfg;
// SMeta operations // SMeta operations
SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
...@@ -54,6 +56,8 @@ STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); ...@@ -54,6 +56,8 @@ STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
SSmaCfg * metaGetSmaInfoByName(SMeta *pMeta, const char *indexName);
STSmaWrapper * metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid);
SMTbCursor *metaOpenTbCursor(SMeta *pMeta); SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
void metaCloseTbCursor(SMTbCursor *pTbCur); void metaCloseTbCursor(SMTbCursor *pTbCur);
...@@ -63,6 +67,10 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); ...@@ -63,6 +67,10 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid);
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); void metaCloseCtbCurosr(SMCtbCursor *pCtbCur);
tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur);
SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid);
void metaCloseSmaCurosr(SMSmaCursor *pSmaCur);
const char * metaSmaCursorNext(SMSmaCursor *pSmaCur);
// Options // Options
void metaOptionsInit(SMetaCfg *pMetaCfg); void metaOptionsInit(SMetaCfg *pMetaCfg);
void metaOptionsClear(SMetaCfg *pMetaCfg); void metaOptionsClear(SMetaCfg *pMetaCfg);
......
...@@ -87,6 +87,27 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); ...@@ -87,6 +87,27 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
int tsdbPrepareCommit(STsdb *pTsdb); int tsdbPrepareCommit(STsdb *pTsdb);
int tsdbCommit(STsdb *pTsdb); int tsdbCommit(STsdb *pTsdb);
/**
* @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
int32_t tsdbInsertTSmaData(STsdb *pTsdb, STSma *param, STSmaData *pData);
/**
* @brief Insert RSma(Time-range-wise Rollup SMA) data.
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
int32_t tsdbInsertRSmaData(STsdb *pTsdb, SRSma *param, STSmaData *pData);
// STsdbCfg // STsdbCfg
int tsdbOptionsInit(STsdbCfg *); int tsdbOptionsInit(STsdbCfg *);
void tsdbOptionsClear(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *);
......
...@@ -213,6 +213,10 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA ...@@ -213,6 +213,10 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA
//} //}
static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) {
if (pHandle->tbIdHash) {
taosHashClear(pHandle->tbIdHash);
}
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (pHandle->tbIdHash == NULL) { if (pHandle->tbIdHash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
...@@ -227,6 +231,23 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S ...@@ -227,6 +231,23 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S
return 0; return 0;
} }
static FORCE_INLINE int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) {
if (pHandle->tbIdHash == NULL) {
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (pHandle->tbIdHash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i);
taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
}
return 0;
}
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlock(STqReadHandle *pHandle);
int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo);
......
...@@ -33,6 +33,8 @@ int metaOpenDB(SMeta* pMeta); ...@@ -33,6 +33,8 @@ int metaOpenDB(SMeta* pMeta);
void metaCloseDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta);
int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg);
int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
int metaSaveSmaToDB(SMeta* pMeta, SSmaCfg* pTbCfg);
int metaRemoveSmaFromDb(SMeta* pMeta, const char* indexName);
// SMetaCache // SMetaCache
int metaOpenCache(SMeta* pMeta); int metaOpenCache(SMeta* pMeta);
...@@ -76,4 +78,4 @@ struct SMeta { ...@@ -76,4 +78,4 @@ struct SMeta {
} }
#endif #endif
#endif /*_TD_META_DEF_H_*/ #endif /*_TD_META_DEF_H_*/
\ No newline at end of file
...@@ -25,43 +25,41 @@ ...@@ -25,43 +25,41 @@
extern "C" { extern "C" {
#endif #endif
extern int32_t tqDebugFlag; #define tqFatal(...) \
{ \
#define tqFatal(...) \ if (tqDebugFlag & DEBUG_FATAL) { \
{ \ taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
if (tqDebugFlag & DEBUG_FATAL) { \ } \
taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \
} \
} }
#define tqError(...) \ #define tqError(...) \
{ \ { \
if (tqDebugFlag & DEBUG_ERROR) { \ if (tqDebugFlag & DEBUG_ERROR) { \
taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \ taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \ } \
} }
#define tqWarn(...) \ #define tqWarn(...) \
{ \ { \
if (tqDebugFlag & DEBUG_WARN) { \ if (tqDebugFlag & DEBUG_WARN) { \
taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \ taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \ } \
} }
#define tqInfo(...) \ #define tqInfo(...) \
{ \ { \
if (tqDebugFlag & DEBUG_INFO) { \ if (tqDebugFlag & DEBUG_INFO) { \
taosPrintLog("TQ ", 255, __VA_ARGS__); \ taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); \
} \ } \
} }
#define tqDebug(...) \ #define tqDebug(...) \
{ \ { \
if (tqDebugFlag & DEBUG_DEBUG) { \ if (tqDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \
} \ } \
} }
#define tqTrace(...) \ #define tqTrace(...) \
{ \ { \
if (tqDebugFlag & DEBUG_TRACE) { \ if (tqDebugFlag & DEBUG_TRACE) { \
taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); \
} \ } \
} }
#define TQ_BUFFER_SIZE 8 #define TQ_BUFFER_SIZE 8
......
...@@ -16,6 +16,10 @@ ...@@ -16,6 +16,10 @@
#ifndef _TD_TSDB_COMMIT_H_ #ifndef _TD_TSDB_COMMIT_H_
#define _TD_TSDB_COMMIT_H_ #define _TD_TSDB_COMMIT_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef struct { typedef struct {
int minFid; int minFid;
int midFid; int midFid;
...@@ -66,4 +70,8 @@ int tsdbApplyRtn(STsdbRepo *pRepo); ...@@ -66,4 +70,8 @@ int tsdbApplyRtn(STsdbRepo *pRepo);
#endif #endif
#ifdef __cplusplus
}
#endif
#endif /* _TD_TSDB_COMMIT_H_ */ #endif /* _TD_TSDB_COMMIT_H_ */
\ No newline at end of file
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
#include "tsdbMemory.h" #include "tsdbMemory.h"
#include "tsdbOptions.h" #include "tsdbOptions.h"
#include "tsdbReadImpl.h" #include "tsdbReadImpl.h"
#include "tsdbSma.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
......
...@@ -18,6 +18,10 @@ ...@@ -18,6 +18,10 @@
#include "tsdbFile.h" #include "tsdbFile.h"
#ifdef __cplusplus
extern "C" {
#endif
// ================== TSDB global config // ================== TSDB global config
extern bool tsdbForceKeepFile; extern bool tsdbForceKeepFile;
...@@ -38,6 +42,7 @@ typedef struct { ...@@ -38,6 +42,7 @@ typedef struct {
typedef struct { typedef struct {
STsdbFSMeta meta; // FS meta STsdbFSMeta meta; // FS meta
SArray * df; // data file array SArray * df; // data file array
SArray * smaf; // sma data file array
} SFSStatus; } SFSStatus;
typedef struct { typedef struct {
...@@ -111,4 +116,8 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { ...@@ -111,4 +116,8 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
return 0; return 0;
} }
#ifdef __cplusplus
}
#endif
#endif /* _TD_TSDB_FS_H_ */ #endif /* _TD_TSDB_FS_H_ */
...@@ -19,6 +19,10 @@ ...@@ -19,6 +19,10 @@
#include "tchecksum.h" #include "tchecksum.h"
#include "tfs.h" #include "tfs.h"
#ifdef __cplusplus
extern "C" {
#endif
#define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_HEAD_SIZE 512
#define TSDB_FILE_DELIMITER 0xF00AFA0F #define TSDB_FILE_DELIMITER 0xF00AFA0F
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
...@@ -410,4 +414,8 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { ...@@ -410,4 +414,8 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
return true; return true;
} }
#ifdef __cplusplus
}
#endif
#endif /* _TS_TSDB_FILE_H_ */ #endif /* _TS_TSDB_FILE_H_ */
\ No newline at end of file
...@@ -18,13 +18,21 @@ ...@@ -18,13 +18,21 @@
#include "tlog.h" #include "tlog.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int32_t tsdbDebugFlag; extern int32_t tsdbDebugFlag;
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) #define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) #define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) #define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) #define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}
#endif
#endif /* _TD_TSDB_LOG_H_ */ #endif /* _TD_TSDB_LOG_H_ */
\ No newline at end of file
...@@ -16,6 +16,10 @@ ...@@ -16,6 +16,10 @@
#ifndef _TD_TSDB_MEMORY_H_ #ifndef _TD_TSDB_MEMORY_H_
#define _TD_TSDB_MEMORY_H_ #define _TD_TSDB_MEMORY_H_
#ifdef __cplusplus
extern "C" {
#endif
static void * taosTMalloc(size_t size); static void * taosTMalloc(size_t size);
static void * taosTCalloc(size_t nmemb, size_t size); static void * taosTCalloc(size_t nmemb, size_t size);
static void * taosTRealloc(void *ptr, size_t size); static void * taosTRealloc(void *ptr, size_t size);
...@@ -70,5 +74,8 @@ static FORCE_INLINE void* taosTZfree(void* ptr) { ...@@ -70,5 +74,8 @@ static FORCE_INLINE void* taosTZfree(void* ptr) {
return NULL; return NULL;
} }
#ifdef __cplusplus
}
#endif
#endif /* _TD_TSDB_MEMORY_H_ */ #endif /* _TD_TSDB_MEMORY_H_ */
\ No newline at end of file
...@@ -24,6 +24,10 @@ ...@@ -24,6 +24,10 @@
#include "tsdbMemory.h" #include "tsdbMemory.h"
#include "tcommon.h" #include "tcommon.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SReadH SReadH; typedef struct SReadH SReadH;
typedef struct { typedef struct {
...@@ -244,4 +248,8 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { ...@@ -244,4 +248,8 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
return 0; return 0;
} }
#ifdef __cplusplus
}
#endif
#endif /*_TD_TSDB_READ_IMPL_H_*/ #endif /*_TD_TSDB_READ_IMPL_H_*/
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TSDB_SMA_H_
#define _TD_TSDB_SMA_H_
// insert/update interface
int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData);
int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData);
// query interface
// TODO: This is the basic params, and should wrap the params to a queryHandle.
int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult);
// management interface
int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void* result);
int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
// internal func
static FORCE_INLINE int32_t tsdbEncodeTSmaKey(uint64_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) {
int32_t len = 0;
len += taosEncodeFixedU64(pData, tableUid);
len += taosEncodeFixedU16(pData, colId);
len += taosEncodeFixedI64(pData, tsKey);
return len;
}
#if 0
typedef struct {
int minFid;
int midFid;
int maxFid;
TSKEY minKey;
} SRtn;
typedef struct {
uint64_t uid;
int64_t offset;
int64_t size;
} SKVRecord;
void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
if (key < 0) {
return (int)((key + 1) / tsTickPerDay[precision] / days - 1);
} else {
return (int)((key / tsTickPerDay[precision] / days));
}
}
static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
if (fid >= pRtn->maxFid) {
return 0;
} else if (fid >= pRtn->midFid) {
return 1;
} else if (fid >= pRtn->minFid) {
return 2;
} else {
return -1;
}
}
#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
void *tsdbCommitData(STsdbRepo *pRepo);
int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn);
int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, SBlockIdx *pIdx);
int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock,
bool isLast, bool isSuper, void **ppBuf, void **ppCBuf);
int tsdbApplyRtn(STsdbRepo *pRepo);
#endif
#endif /* _TD_TSDB_SMA_H_ */
\ No newline at end of file
...@@ -88,44 +88,41 @@ int vnodeScheduleTask(SVnodeTask* task); ...@@ -88,44 +88,41 @@ int vnodeScheduleTask(SVnodeTask* task);
int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq);
void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq);
// For Log #define vFatal(...) \
extern int32_t vDebugFlag; do { \
if (vDebugFlag & DEBUG_FATAL) { \
#define vFatal(...) \ taosPrintLog("VND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
do { \ } \
if (vDebugFlag & DEBUG_FATAL) { \
taosPrintLog("VND FATAL ", 255, __VA_ARGS__); \
} \
} while (0) } while (0)
#define vError(...) \ #define vError(...) \
do { \ do { \
if (vDebugFlag & DEBUG_ERROR) { \ if (vDebugFlag & DEBUG_ERROR) { \
taosPrintLog("VND ERROR ", 255, __VA_ARGS__); \ taosPrintLog("VND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define vWarn(...) \ #define vWarn(...) \
do { \ do { \
if (vDebugFlag & DEBUG_WARN) { \ if (vDebugFlag & DEBUG_WARN) { \
taosPrintLog("VND WARN ", 255, __VA_ARGS__); \ taosPrintLog("VND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define vInfo(...) \ #define vInfo(...) \
do { \ do { \
if (vDebugFlag & DEBUG_INFO) { \ if (vDebugFlag & DEBUG_INFO) { \
taosPrintLog("VND ", 255, __VA_ARGS__); \ taosPrintLog("VND ", DEBUG_INFO, 255, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define vDebug(...) \ #define vDebug(...) \
do { \ do { \
if (vDebugFlag & DEBUG_DEBUG) { \ if (vDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \ taosPrintLog("VND ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define vTrace(...) \ #define vTrace(...) \
do { \ do { \
if (vDebugFlag & DEBUG_TRACE) { \ if (vDebugFlag & DEBUG_TRACE) { \
taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \ taosPrintLog("VND ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
// vnodeCfg.h // vnodeCfg.h
......
...@@ -38,11 +38,14 @@ struct SMetaDB { ...@@ -38,11 +38,14 @@ struct SMetaDB {
// DB // DB
DB *pTbDB; DB *pTbDB;
DB *pSchemaDB; DB *pSchemaDB;
DB *pSmaDB;
// IDX // IDX
DB *pNameIdx; DB *pNameIdx;
DB *pStbIdx; DB *pStbIdx;
DB *pNtbIdx; DB *pNtbIdx;
DB *pCtbIdx; DB *pCtbIdx;
DB *pSmaIdx;
// ENV // ENV
DB_ENV *pEvn; DB_ENV *pEvn;
}; };
...@@ -61,11 +64,12 @@ static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT ...@@ -61,11 +64,12 @@ static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT
static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg);
static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
static void metaClearTbCfg(STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg);
static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW); static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
static void metaDBWLock(SMetaDB *pDB); static void metaDBWLock(SMetaDB *pDB);
static void metaDBRLock(SMetaDB *pDB); static void metaDBRLock(SMetaDB *pDB);
static void metaDBULock(SMetaDB *pDB); static void metaDBULock(SMetaDB *pDB);
...@@ -100,6 +104,11 @@ int metaOpenDB(SMeta *pMeta) { ...@@ -100,6 +104,11 @@ int metaOpenDB(SMeta *pMeta) {
return -1; return -1;
} }
if (metaOpenBDBDb(&(pDB->pSmaDB), pDB->pEvn, "sma.db", false) < 0) {
metaCloseDB(pMeta);
return -1;
}
// Open Indices // Open Indices
if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "name.index", pDB->pTbDB, &metaNameIdxCb, false) < 0) { if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "name.index", pDB->pTbDB, &metaNameIdxCb, false) < 0) {
metaCloseDB(pMeta); metaCloseDB(pMeta);
...@@ -121,15 +130,22 @@ int metaOpenDB(SMeta *pMeta) { ...@@ -121,15 +130,22 @@ int metaOpenDB(SMeta *pMeta) {
return -1; return -1;
} }
if (metaOpenBDBIdx(&(pDB->pSmaIdx), pDB->pEvn, "sma.index", pDB->pSmaDB, &metaSmaIdxCb, true) < 0) {
metaCloseDB(pMeta);
return -1;
}
return 0; return 0;
} }
void metaCloseDB(SMeta *pMeta) { void metaCloseDB(SMeta *pMeta) {
if (pMeta->pDB) { if (pMeta->pDB) {
metaCloseBDBIdx(pMeta->pDB->pSmaIdx);
metaCloseBDBIdx(pMeta->pDB->pCtbIdx); metaCloseBDBIdx(pMeta->pDB->pCtbIdx);
metaCloseBDBIdx(pMeta->pDB->pNtbIdx); metaCloseBDBIdx(pMeta->pDB->pNtbIdx);
metaCloseBDBIdx(pMeta->pDB->pStbIdx); metaCloseBDBIdx(pMeta->pDB->pStbIdx);
metaCloseBDBIdx(pMeta->pDB->pNameIdx); metaCloseBDBIdx(pMeta->pDB->pNameIdx);
metaCloseBDBDb(pMeta->pDB->pSmaDB);
metaCloseBDBDb(pMeta->pDB->pSchemaDB); metaCloseBDBDb(pMeta->pDB->pSchemaDB);
metaCloseBDBDb(pMeta->pDB->pTbDB); metaCloseBDBDb(pMeta->pDB->pTbDB);
metaCloseBDBEnv(pMeta->pDB->pEvn); metaCloseBDBEnv(pMeta->pDB->pEvn);
...@@ -142,7 +158,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { ...@@ -142,7 +158,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
tb_uid_t uid; tb_uid_t uid;
char buf[512]; char buf[512];
char buf1[512]; char buf1[512];
void * pBuf; void *pBuf;
DBT key1, value1; DBT key1, value1;
DBT key2, value2; DBT key2, value2;
SSchema *pSchema = NULL; SSchema *pSchema = NULL;
...@@ -210,6 +226,49 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { ...@@ -210,6 +226,49 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
return 0; return 0;
} }
int metaSaveSmaToDB(SMeta *pMeta, SSmaCfg *pSmaCfg) {
char buf[512] = {0}; // TODO: may overflow
void *pBuf = NULL;
DBT key1 = {0}, value1 = {0};
{
// save sma info
pBuf = buf;
key1.data = pSmaCfg->indexName;
key1.size = strlen(key1.data);
tEncodeTSma(&pBuf, pSmaCfg);
value1.data = buf;
value1.size = POINTER_DISTANCE(pBuf, buf);
value1.app_data = pSmaCfg;
}
metaDBWLock(pMeta->pDB);
pMeta->pDB->pSmaDB->put(pMeta->pDB->pSmaDB, NULL, &key1, &value1, 0);
metaDBULock(pMeta->pDB);
return 0;
}
int metaRemoveSmaFromDb(SMeta *pMeta, const char *indexName) {
// TODO
#if 0
DBT key = {0};
key.data = (void *)indexName;
key.size = strlen(indexName);
metaDBWLock(pMeta->pDB);
// TODO: No guarantee of consistence.
// Use transaction or DB->sync() for some guarantee.
pMeta->pDB->pSmaDB->del(pMeta->pDB->pSmaDB, NULL, &key, 0);
metaDBULock(pMeta->pDB);
#endif
return 0;
}
/* ------------------------ STATIC METHODS ------------------------ */ /* ------------------------ STATIC METHODS ------------------------ */
static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
int tlen = 0; int tlen = 0;
...@@ -394,7 +453,7 @@ static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey ...@@ -394,7 +453,7 @@ static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey
static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) {
STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); STbCfg *pTbCfg = (STbCfg *)(pValue->app_data);
DBT * pDbt; DBT *pDbt;
if (pTbCfg->type == META_CHILD_TABLE) { if (pTbCfg->type == META_CHILD_TABLE) {
// pDbt = calloc(2, sizeof(DBT)); // pDbt = calloc(2, sizeof(DBT));
...@@ -425,6 +484,16 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey ...@@ -425,6 +484,16 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey
} }
} }
static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) {
SSmaCfg *pSmaCfg = (SSmaCfg *)(pValue->app_data);
memset(pSKey, 0, sizeof(*pSKey));
pSKey->data = &(pSmaCfg->tableUid);
pSKey->size = sizeof(pSmaCfg->tableUid);
return 0;
}
static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) {
int tsize = 0; int tsize = 0;
...@@ -479,7 +548,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) { ...@@ -479,7 +548,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) {
/* ------------------------ FOR QUERY ------------------------ */ /* ------------------------ FOR QUERY ------------------------ */
STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
STbCfg * pTbCfg = NULL; STbCfg *pTbCfg = NULL;
SMetaDB *pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
DBT key = {0}; DBT key = {0};
DBT value = {0}; DBT value = {0};
...@@ -509,7 +578,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { ...@@ -509,7 +578,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
} }
STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
STbCfg * pTbCfg = NULL; STbCfg *pTbCfg = NULL;
SMetaDB *pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
DBT key = {0}; DBT key = {0};
DBT pkey = {0}; DBT pkey = {0};
...@@ -540,13 +609,43 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { ...@@ -540,13 +609,43 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
return pTbCfg; return pTbCfg;
} }
SSmaCfg *metaGetSmaInfoByName(SMeta *pMeta, const char *indexName) {
SSmaCfg *pCfg = NULL;
SMetaDB *pDB = pMeta->pDB;
DBT key = {0};
DBT value = {0};
int ret;
// Set key/value
key.data = (void *)indexName;
key.size = strlen(indexName);
// Query
metaDBRLock(pDB);
ret = pDB->pTbDB->get(pDB->pSmaDB, NULL, &key, &value, 0);
metaDBULock(pDB);
if (ret != 0) {
return NULL;
}
// Decode
pCfg = (SSmaCfg *)malloc(sizeof(SSmaCfg));
if (pCfg == NULL) {
return NULL;
}
tDecodeTSma(value.data, pCfg);
return pCfg;
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
uint32_t nCols; uint32_t nCols;
SSchemaWrapper *pSW = NULL; SSchemaWrapper *pSW = NULL;
SMetaDB * pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
int ret; int ret;
void * pBuf; void *pBuf;
SSchema * pSchema; SSchema *pSchema;
SSchemaKey schemaKey = {uid, sver, 0}; SSchemaKey schemaKey = {uid, sver, 0};
DBT key = {0}; DBT key = {0};
DBT value = {0}; DBT value = {0};
...@@ -578,7 +677,7 @@ struct SMTbCursor { ...@@ -578,7 +677,7 @@ struct SMTbCursor {
SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
SMTbCursor *pTbCur = NULL; SMTbCursor *pTbCur = NULL;
SMetaDB * pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur));
if (pTbCur == NULL) { if (pTbCur == NULL) {
...@@ -609,7 +708,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { ...@@ -609,7 +708,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
DBT key = {0}; DBT key = {0};
DBT value = {0}; DBT value = {0};
STbCfg tbCfg; STbCfg tbCfg;
void * pBuf; void *pBuf;
for (;;) { for (;;) {
if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) { if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) {
...@@ -631,10 +730,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { ...@@ -631,10 +730,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
STSchemaBuilder sb; STSchemaBuilder sb;
STSchema * pTSchema = NULL; STSchema *pTSchema = NULL;
SSchema * pSchema; SSchema *pSchema;
SSchemaWrapper *pSW; SSchemaWrapper *pSW;
STbCfg * pTbCfg; STbCfg *pTbCfg;
tb_uid_t quid; tb_uid_t quid;
pTbCfg = metaGetTbInfoByUid(pMeta, uid); pTbCfg = metaGetTbInfoByUid(pMeta, uid);
...@@ -662,13 +761,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { ...@@ -662,13 +761,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
} }
struct SMCtbCursor { struct SMCtbCursor {
DBC * pCur; DBC *pCur;
tb_uid_t suid; tb_uid_t suid;
}; };
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
SMCtbCursor *pCtbCur = NULL; SMCtbCursor *pCtbCur = NULL;
SMetaDB * pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
int ret; int ret;
pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur));
...@@ -700,7 +799,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { ...@@ -700,7 +799,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
DBT skey = {0}; DBT skey = {0};
DBT pkey = {0}; DBT pkey = {0};
DBT pval = {0}; DBT pval = {0};
void * pBuf; void *pBuf;
STbCfg tbCfg; STbCfg tbCfg;
// Set key // Set key
...@@ -718,6 +817,105 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { ...@@ -718,6 +817,105 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
} }
} }
struct SMSmaCursor {
DBC *pCur;
tb_uid_t uid;
};
SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
SMSmaCursor *pCur = NULL;
SMetaDB *pDB = pMeta->pDB;
int ret;
pCur = (SMSmaCursor *)calloc(1, sizeof(*pCur));
if (pCur == NULL) {
return NULL;
}
pCur->uid = uid;
// TODO: lock?
ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0);
if (ret != 0) {
free(pCur);
return NULL;
}
return pCur;
}
void metaCloseSmaCurosr(SMSmaCursor *pCur) {
if (pCur) {
if (pCur->pCur) {
pCur->pCur->close(pCur->pCur);
}
free(pCur);
}
}
const char *metaSmaCursorNext(SMSmaCursor *pCur) {
DBT skey = {0};
DBT pkey = {0};
DBT pval = {0};
// Set key
skey.data = &(pCur->uid);
skey.size = sizeof(pCur->uid);
// TODO: lock?
if (pCur->pCur->pget(pCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) {
const char *indexName = (const char *)pkey.data;
assert(indexName != NULL);
return indexName;
} else {
return NULL;
}
}
STSmaWrapper *metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid) {
STSmaWrapper *pSW = NULL;
pSW = calloc(sizeof(*pSW), 1);
if (pSW == NULL) {
return NULL;
}
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
if (pCur == NULL) {
free(pSW);
return NULL;
}
DBT skey = {.data = &(pCur->uid)};
DBT pval = {.size = sizeof(pCur->uid)};
void *pBuf = NULL;
while (true) {
// TODO: lock?
if (pCur->pCur->pget(pCur->pCur, &skey, NULL, &pval, DB_NEXT) == 0) {
++pSW->number;
STSma *tptr = (STSma *)realloc(pSW->tSma, pSW->number * sizeof(STSma));
if (tptr == NULL) {
metaCloseSmaCurosr(pCur);
tdDestroyTSmaWrapper(pSW, true);
return NULL;
}
pSW->tSma = tptr;
pBuf = pval.data;
if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
metaCloseSmaCurosr(pCur);
tdDestroyTSmaWrapper(pSW, true);
return NULL;
}
continue;
}
break;
}
metaCloseSmaCurosr(pCur);
return pSW;
}
static void metaDBWLock(SMetaDB *pDB) { static void metaDBWLock(SMetaDB *pDB) {
#if IMPL_WITH_LOCK #if IMPL_WITH_LOCK
pthread_rwlock_wrlock(&(pDB->rwlock)); pthread_rwlock_wrlock(&(pDB->rwlock));
......
...@@ -106,3 +106,20 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) { ...@@ -106,3 +106,20 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
// TODO // TODO
return 0; return 0;
} }
int metaCreateSma(SMeta *pMeta, SSmaCfg *pSmaCfg) {
// Validate the tbOptions
// if (metaValidateTbCfg(pMeta, pTbCfg) < 0) {
// // TODO: handle error
// return -1;
// }
// TODO: add atomicity
if (metaSaveSmaToDB(pMeta, pSmaCfg) < 0) {
// TODO: handle error
return -1;
}
return 0;
}
...@@ -72,6 +72,8 @@ void tqClose(STQ* pTq) { ...@@ -72,6 +72,8 @@ void tqClose(STQ* pTq) {
} }
int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
// if waiting
// memcpy and send msg to fetch thread
// TODO: add reference // TODO: add reference
// if handle waiting, launch query and response to consumer // if handle waiting, launch query and response to consumer
// //
...@@ -210,7 +212,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ...@@ -210,7 +212,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont; SMqConsumeReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId; int64_t consumerId = pReq->consumerId;
int64_t fetchOffset; int64_t fetchOffset;
/*int64_t blockingTime = pReq->blockingTime;*/ int64_t blockingTime = pReq->blockingTime;
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
fetchOffset = 0; fetchOffset = 0;
......
...@@ -1054,7 +1054,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { ...@@ -1054,7 +1054,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
// pfs->metaCacheComp = NULL; // pfs->metaCacheComp = NULL;
// } else { // } else {
// // remove meta.tmp file // // remove meta.tmp file
// remove(mf.f.aname); // taosRemoveFile(mf.f.aname);
// taosHashCleanup(pfs->metaCacheComp); // taosHashCleanup(pfs->metaCacheComp);
// pfs->metaCacheComp = NULL; // pfs->metaCacheComp = NULL;
// } // }
......
...@@ -439,7 +439,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { ...@@ -439,7 +439,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
remove(tfname); taosRemoveFile(tfname);
return -1; return -1;
} }
...@@ -447,7 +447,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { ...@@ -447,7 +447,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (fsheader.len > 0) { if (fsheader.len > 0) {
if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
taosCloseFile(&pFile); taosCloseFile(&pFile);
remove(tfname); taosRemoveFile(tfname);
return -1; return -1;
} }
...@@ -458,7 +458,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { ...@@ -458,7 +458,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
(void)remove(tfname); (void)taosRemoveFile(tfname);
taosTZfree(pBuf); taosTZfree(pBuf);
return -1; return -1;
} }
...@@ -468,7 +468,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { ...@@ -468,7 +468,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosFsyncFile(pFile) < 0) { if (taosFsyncFile(pFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
remove(tfname); taosRemoveFile(tfname);
taosTZfree(pBuf); taosTZfree(pBuf);
return -1; return -1;
} }
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdbDef.h"
#define SMA_STORAGE_TSDB_DAYS 30
#define SMA_STORAGE_SPLIT_HOURS 24
#define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8
#define SMA_STORE_SINGLE_BLOCKS // store SMA data by single block or multiple blocks
typedef enum {
SMA_STORAGE_LEVEL_TSDB = 0, // store TSma in dir e.g. vnode${N}/tsdb/.tsma
SMA_STORAGE_LEVEL_DFILESET = 1 // store TSma in file e.g. vnode${N}/tsdb/v2f1900.tsma.${sma_index_name}
} ESmaStorageLevel;
typedef struct {
STsdb * pTsdb;
char * pDFile; // TODO: use the real DFile type, not char*
int32_t interval; // interval with the precision of DB
int32_t blockSize; // size of SMA block item
// TODO
} STSmaWriteH;
typedef struct {
int32_t iter;
} SmaFsIter;
typedef struct {
STsdb * pTsdb;
char * pDFile; // TODO: use the real DFile type, not char*
int32_t interval; // interval with the precision of DB
int32_t blockSize; // size of SMA block item
int8_t storageLevel;
int8_t days;
SmaFsIter smaFsIter;
// TODO
} STSmaReadH;
// declaration of static functions
static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit);
static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, int32_t sectionDataLen, int32_t nBlocks);
static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen);
static int32_t tsdbTSmaDataSplit(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t days, int32_t nOffset,
int32_t fid, int32_t *nSmaBlocks);
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision);
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel,
int32_t fid);
static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin);
static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin);
/**
* @brief Judge the tSma storage level
*
* @param interval
* @param intervalUnit
* @return int32_t
*/
static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit) {
// TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
switch (intervalUnit) {
case TD_TIME_UNIT_HOUR:
if (interval < SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_MINUTE:
if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_SEC:
if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_MILLISEC:
if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_MICROSEC:
if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_NANOSEC:
if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
default:
break;
}
return SMA_STORAGE_LEVEL_TSDB;
}
/**
* @brief Insert TSma data blocks to B+Tree
*
* @param bTree
* @param smaKey
* @param pData
* @param dataLen
* @return int32_t
*/
static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen) {
// TODO: insert sma data blocks into B+Tree
printf("insert sma data blocks into B+Tree: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d\n",
*(uint64_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
return TSDB_CODE_SUCCESS;
}
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision) {
if (intervalUnit < TD_TIME_UNIT_MILLISEC) {
switch (intervalUnit) {
case TD_TIME_UNIT_YEAR:
case TD_TIME_UNIT_SEASON:
case TD_TIME_UNIT_MONTH:
case TD_TIME_UNIT_WEEK:
// illegal time unit
tsdbError("invalid interval unit: %d\n", intervalUnit);
TASSERT(0);
break;
case TD_TIME_UNIT_DAY: // the interval for tSma calculation must <= day
interval *= 86400 * 1e3;
break;
case TD_TIME_UNIT_HOUR:
interval *= 3600 * 1e3;
break;
case TD_TIME_UNIT_MINUTE:
interval *= 60 * 1e3;
break;
case TD_TIME_UNIT_SEC:
interval *= 1e3;
break;
default:
break;
}
}
switch (intervalUnit) {
case TD_TIME_UNIT_MILLISEC:
if (TSDB_TIME_PRECISION_MILLI == precision) {
return interval;
} else if (TSDB_TIME_PRECISION_MICRO == precision) {
return interval * 1e3;
} else { // nano second
return interval * 1e6;
}
break;
case TD_TIME_UNIT_MICROSEC:
if (TSDB_TIME_PRECISION_MILLI == precision) {
return interval / 1e3;
} else if (TSDB_TIME_PRECISION_MICRO == precision) {
return interval;
} else { // nano second
return interval * 1e3;
}
break;
case TD_TIME_UNIT_NANOSEC:
if (TSDB_TIME_PRECISION_MILLI == precision) {
return interval / 1e6;
} else if (TSDB_TIME_PRECISION_MICRO == precision) {
return interval / 1e3;
} else { // nano second
return interval;
}
break;
default:
if (TSDB_TIME_PRECISION_MILLI == precision) {
return interval * 1e3;
} else if (TSDB_TIME_PRECISION_MICRO == precision) {
return interval * 1e6;
} else { // nano second
return interval * 1e9;
}
break;
}
return interval;
}
/**
* @brief Split the TSma data blocks into expected size and insert into B+Tree.
*
* @param pSmaH
* @param pData
* @param nOffset The nOffset of blocks since fid changes.
* @param nBlocks The nBlocks with the same fid since nOffset.
* @return int32_t
*/
static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, int32_t nOffset, int32_t nBlocks) {
STsdb *pTsdb = pSmaH->pTsdb;
TASSERT(pData->colIds != NULL);
tsdbDebug("tsdbInsertTSmaDataSection: nOffset %d, nBlocks %d", nOffset, nBlocks);
printf("tsdbInsertTSmaDataSection: nOffset %d, nBlocks %d\n", nOffset, nBlocks);
int32_t colDataLen = pData->dataLen / pData->numOfColIds;
int32_t sectionDataLen = pSmaH->blockSize * nBlocks;
for (col_id_t i = 0; i < pData->numOfColIds; ++i) {
// param: pointer of B+Tree, key, value, dataLen
void *bTree = pSmaH->pDFile;
#ifndef SMA_STORE_SINGLE_BLOCKS
// save tSma data blocks as a whole
char smaKey[SMA_KEY_LEN] = {0};
void *pSmaKey = &smaKey;
tsdbEncodeTSmaKey(pData->tableUid, *(pData->colIds + i), pData->tsWindow.skey + nOffset * pSmaH->interval,
(void **)&pSmaKey);
if (tsdbInsertTSmaBlocks(bTree, smaKey, pData->data + i * colDataLen + nOffset * pSmaH->blockSize, sectionDataLen) <
0) {
tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
#else
// save tSma data blocks separately
for (int32_t n = 0; n < nBlocks; ++n) {
char smaKey[SMA_KEY_LEN] = {0};
void *pSmaKey = &smaKey;
tsdbEncodeTSmaKey(pData->tableUid, *(pData->colIds + i), pData->tsWindow.skey + (nOffset + n) * pSmaH->interval,
(void **)&pSmaKey);
if (tsdbInsertTSmaBlocks(bTree, smaKey, pData->data + i * colDataLen + (nOffset + n) * pSmaH->blockSize,
pSmaH->blockSize) < 0) {
tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
}
#endif
}
return TSDB_CODE_SUCCESS;
}
static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) {
pSmaH->pTsdb = pTsdb;
pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision);
pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
}
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel,
int32_t fid) {
// TODO
pSmaH->pDFile = "tSma_interval_file_name";
return TSDB_CODE_SUCCESS;
} /**
* @brief Split the sma data blocks by fid.
*
* @param pSmaH
* @param param
* @param pData
* @param nOffset
* @param fid
* @param nSmaBlocks
* @return int32_t
*/
static int32_t tsdbTSmaDataSplit(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t days, int32_t nOffset,
int32_t fid, int32_t *nSmaBlocks) {
STsdbCfg *pCfg = REPO_CFG(pSmaH->pTsdb);
// TODO: use binary search
for (int32_t n = nOffset + 1; n < pData->numOfBlocks; ++n) {
// TODO: The tsWindow.skey should use the precision of DB.
int32_t tFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey + pSmaH->interval * n, days, pCfg->precision));
if (tFid > fid) {
*nSmaBlocks = n - nOffset;
break;
}
}
return TSDB_CODE_SUCCESS;
}
/**
* @brief Insert/Update Time-range-wise SMA data.
* - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g.
* v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files.
* - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The
* days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d).
* - The destination file of one data block for some interval is determined by its start TS key.
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData) {
STsdbCfg * pCfg = REPO_CFG(pTsdb);
STSmaData * curData = pData;
STSmaWriteH tSmaH = {0};
tsdbInitTSmaWriteH(&tSmaH, pTsdb, param, pData);
if (pData->numOfBlocks <= 0 || pData->numOfColIds <= 0 || pData->dataLen <= 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
// Step 1: Judge the storage level
int32_t storageLevel = tsdbJudgeStorageLevel(param->interval, param->intervalUnit);
int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile;
// Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
// - Set and open the DFile or the B+Tree file
int32_t minFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey, daysPerFile, pCfg->precision));
int32_t maxFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.ekey, daysPerFile, pCfg->precision));
if (minFid == maxFid) {
// Save all the TSma data to one file
// TODO: tsdbStartTSmaCommit();
tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, minFid);
tsdbInsertTSmaDataSection(&tSmaH, pData, 0, pData->numOfBlocks);
// TODO:tsdbEndTSmaCommit();
} else if (minFid < maxFid) {
// Split the TSma data and save to multiple files. As there is limit for the span, it can't span more than 2 files
// actually.
// TODO: tsdbStartTSmaCommit();
int32_t tFid = minFid;
int32_t nOffset = 0;
int32_t nSmaBlocks = 0;
do {
tsdbTSmaDataSplit(&tSmaH, param, pData, daysPerFile, nOffset, tFid, &nSmaBlocks);
tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, tFid);
if (tsdbInsertTSmaDataSection(&tSmaH, pData, nOffset, nSmaBlocks) < 0) {
return terrno;
}
++tFid;
nOffset += nSmaBlocks;
if (tFid == maxFid) {
tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, tFid);
tsdbInsertTSmaDataSection(&tSmaH, pData, nOffset, pData->numOfBlocks - nOffset);
break;
}
} while (true);
// TODO:tsdbEndTSmaCommit();
} else {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, SRSma *param, STSmaData *pData, int32_t fid) {
// TODO
pSmaH->pDFile = "rSma_interval_file_name";
return TSDB_CODE_SUCCESS;
}
int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData) {
STsdbCfg * pCfg = REPO_CFG(pTsdb);
STSma * tParam = &param->tsma;
STSmaData * curData = pData;
STSmaWriteH tSmaH = {0};
tsdbInitTSmaWriteH(&tSmaH, pTsdb, tParam, pData);
int32_t nSmaBlocks = pData->numOfBlocks;
int32_t colDataLen = pData->dataLen / nSmaBlocks;
// Step 2.2: Storage of SMA_STORAGE_LEVEL_DFILESET
// TODO: Use the daysPerFile for rSma data, not for TS data.
// TODO: The lifecycle of rSma data should be processed like the TS data files.
int32_t minFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey, pCfg->daysPerFile, pCfg->precision));
int32_t maxFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.ekey, pCfg->daysPerFile, pCfg->precision));
if (minFid == maxFid) {
// Save all the TSma data to one file
tsdbSetRSmaDataFile(&tSmaH, param, pData, minFid);
// TODO: tsdbStartTSmaCommit();
tsdbInsertTSmaDataSection(&tSmaH, pData, colDataLen, nSmaBlocks);
// TODO:tsdbEndTSmaCommit();
} else if (minFid < maxFid) {
// Split the TSma data and save to multiple files. As there is limit for the span, it can't span more than 2 files
// actually.
// TODO: tsdbStartTSmaCommit();
int32_t tmpFid = 0;
int32_t step = 0;
for (int32_t n = 0; n < pData->numOfBlocks; ++n) {
}
tsdbInsertTSmaDataSection(&tSmaH, pData, colDataLen, nSmaBlocks);
// TODO:tsdbEndTSmaCommit();
} else {
TASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
// Step 4: finish
return TSDB_CODE_SUCCESS;
}
/**
* @brief Init of tSma ReadH
*
* @param pSmaH
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) {
pSmaH->pTsdb = pTsdb;
pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision);
pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
}
/**
* @brief Init of tSma FS
*
* @param pReadH
* @param param
* @param queryWin
* @return int32_t
*/
static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin) {
int32_t storageLevel = tsdbJudgeStorageLevel(param->interval, param->intervalUnit);
int32_t daysPerFile =
storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : REPO_CFG(pReadH->pTsdb)->daysPerFile;
pReadH->storageLevel = storageLevel;
pReadH->days = daysPerFile;
pReadH->smaFsIter.iter = 0;
}
/**
* @brief Set and open tSma file if it has key locates in queryWin.
*
* @param pReadH
* @param param
* @param queryWin
* @return true
* @return false
*/
static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin) {
SArray *smaFs = pReadH->pTsdb->fs->cstatus->smaf;
int32_t nSmaFs = taosArrayGetSize(smaFs);
pReadH->pDFile = NULL;
while (pReadH->smaFsIter.iter < nSmaFs) {
void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter);
if (pSmaFile) { // match(indexName, queryWindow)
// TODO: select the file by index_name ...
pReadH->pDFile = pSmaFile;
++pReadH->smaFsIter.iter;
break;
}
++pReadH->smaFsIter.iter;
}
if (pReadH->pDFile != NULL) {
tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]");
return true;
}
return false;
}
/**
* @brief Return the data between queryWin and fill the pData.
*
* @param pTsdb
* @param param
* @param pData
* @param queryWin
* @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM.
* @return int32_t
*/
int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult) {
STSmaReadH tReadH = {0};
tsdbInitTSmaReadH(&tReadH, pTsdb, param, pData);
tsdbInitTSmaFile(&tReadH, param, queryWin);
int32_t nResult = 0;
int64_t lastKey = 0;
while (true) {
if (nResult >= nMaxResult) {
break;
}
// set and open the file according to the STSma param
if (tsdbSetAndOpenTSmaFile(&tReadH, param, queryWin)) {
char bTree[100] = "\0";
while (strncmp(bTree, "has more nodes", 100) == 0) {
if (nResult >= nMaxResult) {
break;
}
// tsdbGetDataFromBTree(bTree, queryWin, lastKey)
// fill the pData
++nResult;
}
}
}
// read data from file and fill the result
return TSDB_CODE_SUCCESS;
}
/**
* @brief Get the start TS key of the last data block of one interval/sliding.
*
* @param pTsdb
* @param param
* @param result
* @return int32_t
* 1) Return 0 and fill the result if the check procedure is normal;
* 2) Return -1 if error occurs during the check procedure.
*/
int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result) {
const char *procedure = "";
if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) {
return -1;
}
// fill the result
return TSDB_CODE_SUCCESS;
}
/**
* @brief Remove the tSma data files related to param between pWin.
*
* @param pTsdb
* @param param
* @param pWin
* @return int32_t
*/
int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin) {
// for ("tSmaFiles of param-interval-sliding between pWin") {
// // remove the tSmaFile
// }
return TSDB_CODE_SUCCESS;
}
\ No newline at end of file
...@@ -15,6 +15,14 @@ ...@@ -15,6 +15,14 @@
#include "tsdbDef.h" #include "tsdbDef.h"
/**
* @brief insert TS data
*
* @param pTsdb
* @param pMsg
* @param pRsp
* @return int
*/
int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
// Check if mem is there. If not, create one. // Check if mem is there. If not, create one.
if (pTsdb->mem == NULL) { if (pTsdb->mem == NULL) {
...@@ -24,4 +32,37 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { ...@@ -24,4 +32,37 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
} }
} }
return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL); return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL);
}
/**
* @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
* TODO: Who is responsible for resource allocate and release?
*/
int32_t tsdbInsertTSmaData(STsdb *pTsdb, STSma *param, STSmaData *pData) {
int32_t code = TSDB_CODE_SUCCESS;
if ((code = tsdbInsertTSmaDataImpl(pTsdb, param, pData)) < 0) {
tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
return code;
}
/**
* @brief Insert Time-range-wise Rollup Sma(RSma) data
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
int32_t tsdbInsertRSmaData(STsdb *pTsdb, SRSma *param, STSmaData *pData) {
int32_t code = TSDB_CODE_SUCCESS;
if ((code = tsdbInsertRSmaDataImpl(pTsdb, param, pData)) < 0) {
tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
return code;
} }
\ No newline at end of file
...@@ -32,6 +32,11 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { ...@@ -32,6 +32,11 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
pLoad->totalStorage = 300; pLoad->totalStorage = 300;
pLoad->compStorage = 200; pLoad->compStorage = 200;
pLoad->pointsWritten = 100; pLoad->pointsWritten = 100;
pLoad->numOfSelectReqs = 1;
pLoad->numOfInsertReqs = 3;
pLoad->numOfInsertSuccessReqs = 2;
pLoad->numOfBatchInsertReqs = 5;
pLoad->numOfBatchInsertSuccessReqs = 4;
return 0; return 0;
} }
......
...@@ -41,10 +41,8 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { ...@@ -41,10 +41,8 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
return 0; return 0;
} }
int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SVCreateTbReq vCreateTbReq; void *ptr = NULL;
SVCreateTbBatchReq vCreateTbBatchReq;
void *ptr = NULL;
if (pVnode->config.streamMode == 0) { if (pVnode->config.streamMode == 0) {
ptr = vnodeMalloc(pVnode, pMsg->contLen); ptr = vnodeMalloc(pVnode, pMsg->contLen);
...@@ -64,7 +62,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ...@@ -64,7 +62,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
} }
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_VND_CREATE_STB: case TDMT_VND_CREATE_STB: {
SVCreateTbReq vCreateTbReq = {0};
tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq);
if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) {
// TODO: handle error // TODO: handle error
...@@ -75,7 +74,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ...@@ -75,7 +74,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
free(vCreateTbReq.stbCfg.pTagSchema); free(vCreateTbReq.stbCfg.pTagSchema);
free(vCreateTbReq.name); free(vCreateTbReq.name);
break; break;
case TDMT_VND_CREATE_TABLE: }
case TDMT_VND_CREATE_TABLE: {
SVCreateTbBatchReq vCreateTbBatchReq = {0};
tDeserializeSVCreateTbBatchReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); tDeserializeSVCreateTbBatchReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq);
for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) { for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) {
SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i);
...@@ -97,14 +98,16 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ...@@ -97,14 +98,16 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray));
taosArrayDestroy(vCreateTbBatchReq.pArray); taosArrayDestroy(vCreateTbBatchReq.pArray);
break; break;
}
case TDMT_VND_ALTER_STB: case TDMT_VND_ALTER_STB: {
SVCreateTbReq vAlterTbReq = {0};
vTrace("vgId:%d, process alter stb req", pVnode->vgId); vTrace("vgId:%d, process alter stb req", pVnode->vgId);
tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq);
free(vCreateTbReq.stbCfg.pSchema); free(vAlterTbReq.stbCfg.pSchema);
free(vCreateTbReq.stbCfg.pTagSchema); free(vAlterTbReq.stbCfg.pTagSchema);
free(vCreateTbReq.name); free(vAlterTbReq.name);
break; break;
}
case TDMT_VND_DROP_STB: case TDMT_VND_DROP_STB:
vTrace("vgId:%d, process drop stb req", pVnode->vgId); vTrace("vgId:%d, process drop stb req", pVnode->vgId);
break; break;
...@@ -129,6 +132,15 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ...@@ -129,6 +132,15 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
} }
} break; } break;
case TDMT_VND_CREATE_SMA: { // timeRangeSMA
// 1. tdCreateSmaMeta(pVnode->pMeta,...);
// 2. tdCreateSmaDataInit();
// 3. tdCreateSmaData
} break;
case TDMT_VND_CANCEL_SMA: { // timeRangeSMA
} break;
case TDMT_VND_DROP_SMA: { // timeRangeSMA
} break;
default: default:
ASSERT(0); ASSERT(0);
break; break;
......
add_executable(tqTest "") MESSAGE(STATUS "vnode unit test")
target_sources(tqTest
PRIVATE
"tqMetaTest.cpp"
)
target_include_directories(tqTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/vnode/tq"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(tqTest # GoogleTest requires at least C++11
tq SET(CMAKE_CXX_STANDARD 11)
gtest_main AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
)
enable_testing() # add_executable(tqTest "")
add_test( # target_sources(tqTest
NAME tq_test # PRIVATE
COMMAND tqTest # "tqMetaTest.cpp"
# )
# target_include_directories(tqTest
# PUBLIC
# "${CMAKE_SOURCE_DIR}/include/server/vnode/tq"
# "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
# )
# target_link_libraries(tqTest
# tq
# gtest_main
# )
# enable_testing()
# add_test(
# NAME tq_test
# COMMAND tqTest
# )
ADD_EXECUTABLE(tsdbSmaTest tsdbSmaTest.cpp)
TARGET_LINK_LIBRARIES(
tsdbSmaTest
PUBLIC os util common vnode gtest_main
) )
TARGET_INCLUDE_DIRECTORIES(
tsdbSmaTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/common"
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../src/inc"
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
\ No newline at end of file
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <taoserror.h>
#include <tglobal.h>
#include <iostream>
#include <metaDef.h>
#include <tmsg.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(testCase, tSmaEncodeDecodeTest) {
// encode
STSma tSma = {0};
tSma.version = 0;
tSma.intervalUnit = TD_TIME_UNIT_DAY;
tSma.interval = 1;
tSma.slidingUnit = TD_TIME_UNIT_HOUR;
tSma.sliding = 0;
tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN);
tSma.tableUid = 1234567890;
tSma.numOfColIds = 2;
tSma.numOfFuncIds = 5; // sum/min/max/avg/last
tSma.colIds = (col_id_t *)calloc(tSma.numOfColIds, sizeof(col_id_t));
tSma.funcIds = (uint16_t *)calloc(tSma.numOfFuncIds, sizeof(uint16_t));
for (int32_t i = 0; i < tSma.numOfColIds; ++i) {
*(tSma.colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID);
}
for (int32_t i = 0; i < tSma.numOfFuncIds; ++i) {
*(tSma.funcIds + i) = (i + 2);
}
STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma};
uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper);
void *buf = calloc(bufLen, 1);
assert(buf != NULL);
STSmaWrapper *pSW = (STSmaWrapper *)buf;
uint32_t len = tEncodeTSmaWrapper(&buf, &tSmaWrapper);
EXPECT_EQ(len, bufLen);
// decode
STSmaWrapper dstTSmaWrapper = {0};
void * result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper);
assert(result != NULL);
EXPECT_EQ(tSmaWrapper.number, dstTSmaWrapper.number);
for (int i = 0; i < tSmaWrapper.number; ++i) {
STSma *pSma = tSmaWrapper.tSma + i;
STSma *qSma = dstTSmaWrapper.tSma + i;
EXPECT_EQ(pSma->version, qSma->version);
EXPECT_EQ(pSma->intervalUnit, qSma->intervalUnit);
EXPECT_EQ(pSma->slidingUnit, qSma->slidingUnit);
EXPECT_STRCASEEQ(pSma->indexName, qSma->indexName);
EXPECT_EQ(pSma->numOfColIds, qSma->numOfColIds);
EXPECT_EQ(pSma->numOfFuncIds, qSma->numOfFuncIds);
EXPECT_EQ(pSma->tableUid, qSma->tableUid);
EXPECT_EQ(pSma->interval, qSma->interval);
EXPECT_EQ(pSma->sliding, qSma->sliding);
for (uint32_t j = 0; j < pSma->numOfColIds; ++j) {
EXPECT_EQ(*(col_id_t *)(pSma->colIds + j), *(col_id_t *)(qSma->colIds + j));
}
for (uint32_t j = 0; j < pSma->numOfFuncIds; ++j) {
EXPECT_EQ(*(uint16_t *)(pSma->funcIds + j), *(uint16_t *)(qSma->funcIds + j));
}
}
// resource release
tdDestroyTSma(&tSma, false);
tdDestroyTSmaWrapper(&dstTSmaWrapper, false);
}
TEST(testCase, tSma_DB_Put_Get_Del_Test) {
const char *smaIndexName1 = "sma_index_test_1";
const char *smaIndexName2 = "sma_index_test_2";
const char *smaTestDir = "./smaTest";
const uint64_t tbUid = 1234567890;
// encode
STSma tSma = {0};
tSma.version = 0;
tSma.intervalUnit = TD_TIME_UNIT_DAY;
tSma.interval = 1;
tSma.slidingUnit = TD_TIME_UNIT_HOUR;
tSma.sliding = 0;
tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN);
tSma.tableUid = tbUid;
tSma.numOfColIds = 2;
tSma.numOfFuncIds = 5; // sum/min/max/avg/last
tSma.colIds = (col_id_t *)calloc(tSma.numOfColIds, sizeof(col_id_t));
tSma.funcIds = (uint16_t *)calloc(tSma.numOfFuncIds, sizeof(uint16_t));
for (int32_t i = 0; i < tSma.numOfColIds; ++i) {
*(tSma.colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID);
}
for (int32_t i = 0; i < tSma.numOfFuncIds; ++i) {
*(tSma.funcIds + i) = (i + 2);
}
SMeta * pMeta = NULL;
SSmaCfg * pSmaCfg = &tSma;
const SMetaCfg *pMetaCfg = &defaultMetaOptions;
taosRemoveDir(smaTestDir);
pMeta = metaOpen(smaTestDir, pMetaCfg, NULL);
assert(pMeta != NULL);
// save index 1
metaSaveSmaToDB(pMeta, pSmaCfg);
tstrncpy(pSmaCfg->indexName, smaIndexName2, TSDB_INDEX_NAME_LEN);
pSmaCfg->version = 1;
pSmaCfg->intervalUnit = TD_TIME_UNIT_HOUR;
pSmaCfg->interval = 1;
pSmaCfg->slidingUnit = TD_TIME_UNIT_MINUTE;
pSmaCfg->sliding = 5;
// save index 2
metaSaveSmaToDB(pMeta, pSmaCfg);
// get value by indexName
SSmaCfg *qSmaCfg = NULL;
qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName1);
assert(qSmaCfg != NULL);
printf("name1 = %s\n", qSmaCfg->indexName);
EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1);
EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid);
tdDestroyTSma(qSmaCfg, true);
qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName2);
assert(qSmaCfg != NULL);
printf("name2 = %s\n", qSmaCfg->indexName);
EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2);
EXPECT_EQ(qSmaCfg->interval, tSma.interval);
tdDestroyTSma(qSmaCfg, true);
// get index name by table uid
SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid);
assert(pSmaCur != NULL);
uint32_t indexCnt = 0;
while (1) {
const char* indexName = metaSmaCursorNext(pSmaCur);
if (indexName == NULL) {
break;
}
printf("indexName = %s\n", indexName);
++indexCnt;
}
EXPECT_EQ(indexCnt, 2);
metaCloseSmaCurosr(pSmaCur);
// get wrapper by table uid
STSmaWrapper *pSW = metaGetSmaInfoByUid(pMeta, tbUid);
assert(pSW != NULL);
EXPECT_EQ(pSW->number, 2);
EXPECT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1);
EXPECT_EQ(pSW->tSma->tableUid, tSma.tableUid);
EXPECT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2);
EXPECT_EQ((pSW->tSma + 1)->tableUid, tSma.tableUid);
// resource release
metaRemoveSmaFromDb(pMeta, smaIndexName1);
metaRemoveSmaFromDb(pMeta, smaIndexName2);
tdDestroyTSma(&tSma, false);
metaClose(pMeta);
}
#if 0
TEST(testCase, tSmaInsertTest) {
STSma tSma = {0};
STSmaData* pSmaData = NULL;
STsdb tsdb = {0};
// init
tSma.intervalUnit = TD_TIME_UNIT_DAY;
tSma.interval = 1;
tSma.numOfFuncIds = 5; // sum/min/max/avg/last
int32_t blockSize = tSma.numOfFuncIds * sizeof(int64_t);
int32_t numOfColIds = 3;
int32_t numOfBlocks = 10;
int32_t dataLen = numOfColIds * numOfBlocks * blockSize;
pSmaData = (STSmaData*)malloc(sizeof(STSmaData) + dataLen);
ASSERT_EQ(pSmaData != NULL, true);
pSmaData->tableUid = 3232329230;
pSmaData->numOfColIds = numOfColIds;
pSmaData->numOfBlocks = numOfBlocks;
pSmaData->dataLen = dataLen;
pSmaData->tsWindow.skey = 1640000000;
pSmaData->tsWindow.ekey = 1645788649;
pSmaData->colIds = (col_id_t*)malloc(sizeof(col_id_t) * numOfColIds);
ASSERT_EQ(pSmaData->colIds != NULL, true);
for (int32_t i = 0; i < numOfColIds; ++i) {
*(pSmaData->colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID);
}
// execute
EXPECT_EQ(tsdbInsertTSmaData(&tsdb, &tSma, pSmaData), TSDB_CODE_SUCCESS);
// release
tdDestroySmaData(pSmaData);
}
#endif
#pragma GCC diagnostic pop
\ No newline at end of file
此差异已折叠。
...@@ -27,13 +27,13 @@ ...@@ -27,13 +27,13 @@
#include "os.h" #include "os.h"
#include "tglobal.h" #include "tglobal.h"
#include "catalog.h" #include "catalog.h"
#include "catalogInt.h"
#include "stub.h" #include "stub.h"
#include "taos.h" #include "taos.h"
#include "tdatablock.h"
#include "tdef.h" #include "tdef.h"
#include "tep.h"
#include "trpc.h" #include "trpc.h"
#include "tvariant.h" #include "tvariant.h"
#include "catalogInt.h"
namespace { namespace {
......
...@@ -68,7 +68,7 @@ typedef struct SResultRow { ...@@ -68,7 +68,7 @@ typedef struct SResultRow {
} SResultRow; } SResultRow;
typedef struct SResultRowInfo { typedef struct SResultRowInfo {
SResultRow *pCurResult; // current active result row info SList* pRows;
SResultRow** pResult; // result list SResultRow** pResult; // result list
// int16_t type:8; // data type for hash key // int16_t type:8; // data type for hash key
int32_t size; // number of result set int32_t size; // number of result set
......
此差异已折叠。
...@@ -95,7 +95,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { ...@@ -95,7 +95,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
} }
int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo* )tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
// traverse to the streamscan node to add this table id // traverse to the streamscan node to add this table id
SOperatorInfo* pInfo = pTaskInfo->pRoot; SOperatorInfo* pInfo = pTaskInfo->pRoot;
...@@ -105,7 +105,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA ...@@ -105,7 +105,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
SStreamBlockScanInfo* pScanInfo = pInfo->info; SStreamBlockScanInfo* pScanInfo = pInfo->info;
if (isAdd) { if (isAdd) {
int32_t code = tqReadHandleSetTbUidList(pScanInfo->readerHandle, tableIdList); int32_t code = tqReadHandleAddTbUidList(pScanInfo->readerHandle, tableIdList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
...@@ -114,4 +114,4 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA ...@@ -114,4 +114,4 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
\ No newline at end of file
...@@ -158,7 +158,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { ...@@ -158,7 +158,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
int64_t st = 0; int64_t st = 0;
st = taosGetTimestampUs(); st = taosGetTimestampUs();
*pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); *pRes = pTaskInfo->pRoot->nextDataFn(pTaskInfo->pRoot, &newgroup);
uint64_t el = (taosGetTimestampUs() - st); uint64_t el = (taosGetTimestampUs() - st);
pTaskInfo->cost.elapsedTime += el; pTaskInfo->cost.elapsedTime += el;
...@@ -179,13 +179,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { ...@@ -179,13 +179,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
return pTaskInfo->code; return pTaskInfo->code;
} }
void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) {
SQInfo* pQInfo = (SQInfo*) qinfo;
assert(pQInfo != NULL);
return pQInfo->rspContext;
}
int32_t qKillTask(qTaskInfo_t qinfo) { int32_t qKillTask(qTaskInfo_t qinfo) {
SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
...@@ -221,7 +214,7 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) { ...@@ -221,7 +214,7 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) { if (pTaskInfo == NULL) {
return TSDB_CODE_QRY_INVALID_QHANDLE; return TSDB_CODE_QRY_INVALID_QHANDLE;
} }
...@@ -235,33 +228,3 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) { ...@@ -235,33 +228,3 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) {
queryCostStatis(pTaskInfo); // print the query cost summary queryCostStatis(pTaskInfo); // print the query cost summary
doDestroyTask(pTaskInfo); doDestroyTask(pTaskInfo);
} }
#if 0
//kill by qid
int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) {
int32_t error = TSDB_CODE_SUCCESS;
void** handle = qAcquireTask(pMgmt, qId);
if(handle == NULL) return terrno;
SQInfo* pQInfo = (SQInfo*)(*handle);
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
qWarn("%s be killed(no memory commit).", pQInfo->qId);
setTaskKilled(pQInfo);
// wait query stop
int32_t loop = 0;
while (pQInfo->owner != 0) {
taosMsleep(waitMs);
if(loop++ > waitCount){
error = TSDB_CODE_FAILED;
break;
}
}
qReleaseTask(pMgmt, (void **)&handle, true);
return error;
}
#endif
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册