Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1187
Star 22018
Fork 4786
T
TDengine
提交 f0e392ae 编写于 作者: H Haojun Liao
浏览文件
操作

[td-13039] merge 3.0.

上级 0ca24f3e 405e9e2c
展开全部 隐藏空白更改
内联 并排
Showing with 6839 addition and 1965 deletion
example/CMakeLists.txt
浏览文件 @ f0e392ae
......@@ -2,11 +2,7 @@ aux_source_directory(src TMQ_DEMO_SRC)
add_executable(tmq ${TMQ_DEMO_SRC})
target_link_libraries(
tmq
PUBLIC taos
#PUBLIC util
#PUBLIC common
#PUBLIC os
tmq taos
)
target_include_directories(
tmq
......
example/src/tmq.c
浏览文件 @ f0e392ae
......@@ -160,7 +160,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
}
while (running) {
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1000);
if (tmqmessage) {
msg_process(tmqmessage);
tmq_message_destroy(tmqmessage);
......
include/client/taos.h
浏览文件 @ f0e392ae
......@@ -224,10 +224,8 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
/* ------------------------TMQ CONSUMER INTERFACE------------------------ */
DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
#if 0
DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq);
DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics);
#endif
DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
#if 0
......
include/common/taosdef.h
浏览文件 @ f0e392ae
......@@ -51,6 +51,10 @@ typedef enum {
} ECheckItemType;
typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig;
typedef enum {
TSDB_STATIS_OK = 0, // statis part exist and load successfully
TSDB_STATIS_NONE = 1, // statis part not exist
} ETsdbStatisStatus;
extern char *qtypeStr[];
......
include/common/tcommon.h
浏览文件 @ f0e392ae
......@@ -49,6 +49,11 @@ enum {
TMQ_CONF__RESET_OFFSET__NONE = -3,
};
enum {
TMQ_MSG_TYPE__POLL_RSP = 0,
TMQ_MSG_TYPE__EP_RSP,
};
typedef struct {
uint32_t numOfTables;
SArray* pGroupList;
......
include/common/tglobal.h
浏览文件 @ f0e392ae
......@@ -52,6 +52,13 @@ extern bool tsEnableSlaveQuery;
extern bool tsPrintAuth;
extern int64_t tsTickPerDay[3];
// monitor
extern bool tsEnableMonitor;
extern int32_t tsMonitorInterval;
extern char tsMonitorFqdn[];
extern uint16_t tsMonitorPort;
extern int32_t tsMonitorMaxLogs;
// query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node
......
include/common/tmsg.h
浏览文件 @ f0e392ae
......@@ -167,6 +167,7 @@ typedef struct {
typedef struct {
char db[TSDB_DB_FNAME_LEN];
int64_t dbId;
int32_t vgVersion;
} SBuildUseDBInput;
......@@ -563,6 +564,7 @@ int32_t tDeserializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp);
typedef struct {
char db[TSDB_DB_FNAME_LEN];
int64_t dbId;
int32_t vgVersion;
} SUseDbReq;
......@@ -737,6 +739,9 @@ typedef struct {
int32_t maxRows;
int32_t commitTime;
int32_t fsyncPeriod;
uint32_t hashBegin;
uint32_t hashEnd;
int8_t hashMethod;
int8_t walLevel;
int8_t precision;
int8_t compression;
......@@ -747,6 +752,7 @@ typedef struct {
int8_t selfIndex;
int8_t streamMode;
SReplica replicas[TSDB_MAX_REPLICA];
} SCreateVnodeReq, SAlterVnodeReq;
int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
......@@ -1380,8 +1386,6 @@ typedef struct SMqCMGetSubEpReq {
char cgroup[TSDB_CONSUMER_GROUP_LEN];
} SMqCMGetSubEpReq;
#pragma pack(pop)
static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pMsg->contLen);
......@@ -1851,6 +1855,12 @@ typedef struct {
SMqTbData* tbData;
} SMqTopicData;
typedef struct {
int8_t mqMsgType;
int32_t code;
int32_t epoch;
} SMqRspHead;
typedef struct {
int64_t consumerId;
SSchemaWrapper* schemas;
......@@ -1867,6 +1877,7 @@ typedef struct {
int64_t consumerId;
int64_t blockingTime;
int32_t epoch;
char cgroup[TSDB_CONSUMER_GROUP_LEN];
int64_t currentOffset;
......@@ -1886,11 +1897,19 @@ typedef struct {
typedef struct {
int64_t consumerId;
int32_t epoch;
char cgroup[TSDB_CONSUMER_GROUP_LEN];
SArray* topics; // SArray<SMqSubTopicEp>
} SMqCMGetSubEpRsp;
struct tmq_message_t {
SMqRspHead head;
union {
SMqConsumeRsp consumeRsp;
SMqCMGetSubEpRsp getEpRsp;
};
void* extra;
};
static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
......@@ -1943,7 +1962,6 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
tlen += taosEncodeFixedI32(buf, pRsp->epoch);
tlen += taosEncodeString(buf, pRsp->cgroup);
int32_t sz = taosArrayGetSize(pRsp->topics);
tlen += taosEncodeFixedI32(buf, sz);
......@@ -1956,7 +1974,6 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu
static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) {
buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
buf = taosDecodeFixedI32(buf, &pRsp->epoch);
buf = taosDecodeStringTo(buf, pRsp->cgroup);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
......@@ -1972,6 +1989,8 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
return buf;
}
#pragma pack(pop)
#ifdef __cplusplus
}
#endif
......
include/common/ttime.h
浏览文件 @ f0e392ae
......@@ -52,6 +52,8 @@ void deltaToUtcInitOnce();
int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
#ifdef __cplusplus
}
#endif
......
include/dnode/mnode/mnode.h
浏览文件 @ f0e392ae
......@@ -16,6 +16,8 @@
#ifndef _TD_MND_H_
#define _TD_MND_H_
#include "monitor.h"
#ifdef __cplusplus
extern "C" {
#endif
......@@ -30,20 +32,6 @@ typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
typedef struct SMnodeLoad {
int64_t numOfDnode;
int64_t numOfMnode;
int64_t numOfVgroup;
int64_t numOfDatabase;
int64_t numOfSuperTable;
int64_t numOfChildTable;
int64_t numOfNormalTable;
int64_t numOfColumn;
int64_t totalPoints;
int64_t totalStorage;
int64_t compStorage;
} SMnodeLoad;
typedef struct {
int32_t dnodeId;
int64_t clusterId;
......@@ -92,13 +80,16 @@ int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption);
void mndDestroy(const char *path);
/**
* @brief Get mnode statistics info.
* @brief Get mnode monitor info.
*
* @param pMnode The mnode object.
* @param pLoad Statistics of the mnode.
* @param pClusterInfo
* @param pVgroupInfo
* @param pGrantInfo
* @return int32_t 0 for success, -1 for failure.
*/
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo);
/**
* @brief Get user authentication info.
......
include/libs/catalog/catalog.h
浏览文件 @ f0e392ae
......@@ -95,7 +95,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle);
*/
void catalogFreeHandle(SCatalog* pCatalog);
int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* version);
int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId);
/**
* Get a DB's all vgroup info.
......
include/libs/index/index.h
浏览文件 @ f0e392ae
......@@ -29,6 +29,12 @@ typedef struct SIndexOpts SIndexOpts;
typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
typedef struct SArray SIndexMultiTerm;
typedef struct SIndex SIndexJson;
typedef struct SIndexTerm SIndexJsonTerm;
typedef struct SIndexOpts SIndexJsonOpts;
typedef struct SIndexMultiTermQuery SIndexJsonMultiTermQuery;
typedef struct SArray SIndexJsonMultiTerm;
typedef enum {
ADD_VALUE, // add index colume value
DEL_VALUE, // delete index column value
......@@ -39,24 +45,108 @@ typedef enum {
} SIndexOperOnColumn;
typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType;
/*
* @param: oper
*
* create multi query
* @param oper (input, relation between querys)
*/
SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper);
void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery);
int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type);
/*
* @param:
* @param:
* destroy multi query
* @param pQuery (input, multi-query-object to be destory)
*/
void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery);
/*
* add query to multi query
* @param pQuery (input, multi-query-object)
* @param term (input, single query term)
* @param type (input, single query type)
* @return error code
*/
int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type);
/*
* open index
* @param opt (input, index opt)
* @param path (input, index path)
* @param index (output, index object)
* @return error code
*/
int indexOpen(SIndexOpts* opt, const char* path, SIndex** index);
/*
* close index
* @param index (input, index to be closed)
* @return error code
*/
int indexOpen(SIndexOpts* opt, const char* path, SIndex** index);
void indexClose(SIndex* index);
int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid);
int indexDelete(SIndex* index, SIndexMultiTermQuery* query);
int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result);
int indexRebuild(SIndex* index, SIndexOpts* opt);
/*
* insert terms into index
* @param index (input, index object)
* @param term (input, terms inserted into index)
* @param uid (input, uid of terms)
* @return error code
*/
int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid);
/*
* delete terms that meet query condition
* @param index (input, index object)
* @param query (input, condition query to deleted)
* @return error code
*/
int indexDelete(SIndex* index, SIndexMultiTermQuery* query);
/*
* search index
* @param index (input, index object)
* @param query (input, multi query condition)
* @param result(output, query result)
* @return error code
*/
int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result);
/*
* rebuild index
* @param index (input, index object)
* @parma opt (input, rebuild index opts)
* @return error code
*/
int indexRebuild(SIndex* index, SIndexOpts* opt);
/*
* open index
* @param opt (input,index json opt)
* @param path (input, index json path)
* @param index (output, index json object)
* @return error code
*/
int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index);
/*
* close index
* @param index (input, index to be closed)
* @return void
*/
void tIndexJsonClose(SIndexJson* index);
/*
* insert terms into index
* @param index (input, index object)
* @param term (input, terms inserted into index)
* @param uid (input, uid of terms)
* @return error code
*/
int tIndexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid);
/*
* search index
* @param index (input, index object)
* @param query (input, multi query condition)
* @param result(output, query result)
* @return error code
*/
int tIndexJsonSearch(SIndexJson* index, SIndexJsonMultiTermQuery* query, SArray* result);
/*
* @param
* @param
......
include/libs/monitor/monitor.h 0 → 100644
浏览文件 @ f0e392ae
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MONITOR_H_
#define _TD_MONITOR_H_
#include "tarray.h"
#include "tdef.h"
#ifdef __cplusplus
extern "C" {
#endif
#define MON_STATUS_LEN 8
#define MON_ROLE_LEN 9
#define MON_VER_LEN 12
#define MON_LOG_LEN 1024
typedef struct {
int32_t dnode_id;
char dnode_ep[TSDB_EP_LEN];
} SMonBasicInfo;
typedef struct {
int32_t dnode_id;
char dnode_ep[TSDB_EP_LEN];
char status[MON_STATUS_LEN];
} SMonDnodeDesc;
typedef struct {
int32_t mnode_id;
char mnode_ep[TSDB_EP_LEN];
char role[MON_ROLE_LEN];
} SMonMnodeDesc;
typedef struct {
char first_ep[TSDB_EP_LEN];
int32_t first_ep_dnode_id;
char version[MON_VER_LEN];
float master_uptime; // day
int32_t monitor_interval; // sec
int32_t vgroups_total;
int32_t vgroups_alive;
int32_t vnodes_total;
int32_t vnodes_alive;
int32_t connections_total;
SArray *dnodes; // array of SMonDnodeDesc
SArray *mnodes; // array of SMonMnodeDesc
} SMonClusterInfo;
typedef struct {
int32_t dnode_id;
char vnode_role[MON_ROLE_LEN];
} SMonVnodeDesc;
typedef struct {
int32_t vgroup_id;
char database_name[TSDB_DB_NAME_LEN];
int32_t tables_num;
char status[MON_STATUS_LEN];
SMonVnodeDesc vnodes[TSDB_MAX_REPLICA];
} SMonVgroupDesc;
typedef struct {
SArray *vgroups; // array of SMonVgroupDesc
} SMonVgroupInfo;
typedef struct {
int32_t expire_time;
int32_t timeseries_used;
int32_t timeseries_total;
} SMonGrantInfo;
typedef struct {
float uptime; // day
float cpu_engine;
float cpu_system;
float cpu_cores;
float mem_engine; // MB
float mem_system; // MB
float mem_total; // MB
float disk_engine; // GB
float disk_used; // GB
float disk_total; // GB
float net_in; // Kb/s
float net_out; // Kb/s
float io_read; // Mb/s
float io_write; // Mb/s
float io_read_disk; // Mb/s
float io_write_disk; // Mb/s
int32_t req_select;
float req_select_rate;
int32_t req_insert;
int32_t req_insert_success;
float req_insert_rate;
int32_t req_insert_batch;
int32_t req_insert_batch_success;
float req_insert_batch_rate;
int32_t errors;
int32_t vnodes_num;
int32_t masters;
int8_t has_mnode;
} SMonDnodeInfo;
typedef struct {
char name[TSDB_FILENAME_LEN];
int32_t level;
SDiskSize size;
} SMonDiskDesc;
typedef struct {
SArray *datadirs; // array of SMonDiskDesc
SMonDiskDesc logdir;
SMonDiskDesc tempdir;
} SMonDiskInfo;
typedef struct {
int64_t ts;
int8_t level;
char content[MON_LOG_LEN];
} SMonLogItem;
typedef struct SMonInfo SMonInfo;
typedef struct {
const char *server;
uint16_t port;
int32_t maxLogs;
} SMonCfg;
int32_t monInit(const SMonCfg *pCfg);
void monCleanup();
void monAddLogItem(SMonLogItem *pItem);
SMonInfo *monCreateMonitorInfo();
void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);
void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo);
void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo);
void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo);
void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo);
void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo);
void monSendReport(SMonInfo *pMonitor);
void monCleanupMonitorInfo(SMonInfo *pMonitor);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MONITOR_H_*/
include/libs/qcom/query.h
浏览文件 @ f0e392ae
......@@ -156,6 +156,8 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
*/
int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo);
int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp);
void initQueryModuleMsgHandle();
const SSchema* tGetTbnameColumnSchema();
......
include/libs/scheduler/scheduler.h
浏览文件 @ f0e392ae
......@@ -23,8 +23,6 @@ extern "C" {
#include "catalog.h"
#include "planner.h"
struct SSchJob;
typedef struct SSchedulerCfg {
uint32_t maxJobNum;
} SSchedulerCfg;
......@@ -72,7 +70,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg);
* @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr
* @return
*/
int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes);
int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, int64_t *pJob, const char* sql, SQueryResult *pRes);
/**
* Process the query job, generated according to the query physical plan.
......@@ -80,7 +78,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, str
* @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr
* @return
*/
int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob);
int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, int64_t *pJob);
/**
* Fetch query result from the remote query executor
......@@ -88,7 +86,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDa
* @param data
* @return
*/
int32_t schedulerFetchRows(struct SSchJob *pJob, void **data);
int32_t schedulerFetchRows(int64_t job, void **data);
/**
......@@ -102,7 +100,7 @@ int32_t schedulerFetchRows(struct SSchJob *pJob, void **data);
* Free the query job
* @param pJob
*/
void schedulerFreeJob(void *pJob);
void schedulerFreeJob(int64_t job);
void schedulerDestroy(void);
......
include/libs/sync/sync.h
浏览文件 @ f0e392ae
......@@ -69,15 +69,15 @@ typedef struct SSyncFSM {
// when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result
// user can do something according to the code and isWeak. for example, write data into tsdb
void (*FpCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code);
// when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result
// user can do something according to the code and isWeak. for example, write data into tsdb
void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code);
// when log entry is updated by a new one, FpRollBackCb is called
// user can do something to roll back. for example, delete data from tsdb, or just ignore it
void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code);
// user should implement this function, use "data" to take snapshot into "snapshot"
int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
......@@ -93,10 +93,10 @@ typedef struct SSyncLogStore {
void* data;
// append one log entry
int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf);
int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SRpcMsg* pBuf);
// get one log entry, user need to free pBuf->data
int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf);
int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SRpcMsg* pBuf);
// update log store commit index with "index"
int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index);
......@@ -135,7 +135,9 @@ typedef struct SSyncInfo {
SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN];
SSyncFSM* pFsm;
int32_t (*FpSendMsg)(void* handle, const SEpSet* pEpSet, SRpcMsg* pMsg);
void* rpcClient;
int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
} SSyncInfo;
......@@ -149,8 +151,8 @@ int64_t syncStart(const SSyncInfo* pSyncInfo);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
// int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
ESyncState syncGetMyRole(int64_t rid);
void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);
......
include/os/osDir.h
浏览文件 @ f0e392ae
......@@ -26,6 +26,7 @@ int32_t taosMkDir(const char *dirname);
void taosRemoveOldFiles(const char *dirname, int32_t keepDays);
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
int32_t taosRealPath(char *dirname, int32_t maxlen);
bool taosIsDir(const char *dirname);
#ifdef __cplusplus
}
......
include/os/osSysinfo.h
浏览文件 @ f0e392ae
......@@ -33,21 +33,25 @@ typedef struct {
SDiskSize size;
} SDiskSpace;
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosGetCpuCores();
void taosGetSystemInfo();
bool taosGetEmail(char *email, int32_t maxLen);
bool taosGetOsReleaseName(char *releaseName, int32_t maxLen);
bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores);
int32_t taosGetCpuCores();
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
bool taosGetTotalSysMemoryKB(uint64_t *kb);
bool taosGetProcMemory(float *memoryUsedMB); //
bool taosGetSysMemory(float *memoryUsedMB); //
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
bool taosReadProcIO(int64_t *rchars, int64_t *wchars);
bool taosGetProcIO(float *readKB, float *writeKB);
bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
bool taosGetBandSpeed(float *bandSpeedKb);
void taosGetDisk();
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
bool taosGetProcMemory(float *memoryUsedMB);
bool taosGetSysMemory(float *memoryUsedMB);
int taosSystem(const char *cmd);
int32_t taosSystem(const char *cmd);
void taosKillSystem();
int32_t taosGetSystemUUID(char *uid, int32_t uidlen);
char * taosGetCmdlineByPID(int pid);
char *taosGetCmdlineByPID(int32_t pid);
void taosSetCoreDump(bool enable);
typedef struct {
......
include/util/tconfig.h
浏览文件 @ f0e392ae
......@@ -82,7 +82,7 @@ typedef struct SConfig {
SConfig *cfgInit();
int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr);
int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs); // SConfigPair
int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair
void cfgCleanup(SConfig *pCfg);
int32_t cfgGetSize(SConfig *pCfg);
......
include/util/thttp.h 0 → 100644
浏览文件 @ f0e392ae
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_HTTP_H_
#define _TD_UTIL_HTTP_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, int32_t contLen);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_UTIL_H_*/
source/client/inc/clientInt.h
浏览文件 @ f0e392ae
......@@ -171,7 +171,7 @@ typedef struct SRequestSendRecvBody {
void* fp;
SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed.
SDataBuf requestMsg;
struct SSchJob* pQueryJob; // query job, created according to sql query DAG.
int64_t queryJob; // query job, created according to sql query DAG.
struct SQueryDag* pDag; // the query dag, generated according to the sql statement.
SReqResultInfo resInfo;
} SRequestSendRecvBody;
......
source/client/src/clientHb.c
浏览文件 @ f0e392ae
......@@ -13,19 +13,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "clientInt.h"
#include "trpc.h"
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "trpc.h"
static SClientHbMgr clientHbMgr = {0};
static int32_t hbCreateThread();
static void hbStopThread();
static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) {
return 0;
}
static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
int32_t code = 0;
......@@ -39,8 +37,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray);
for (int32_t i = 0; i < numOfBatchs; ++i) {
SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i);
tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%"PRIx64, rsp->db, rsp->vgVersion, rsp->uid);
tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->uid);
if (rsp->vgVersion < 0) {
code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid);
} else {
......@@ -60,8 +58,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
taosHashCleanup(vgInfo.vgHash);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
}
}
catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo);
}
......@@ -106,8 +104,8 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
return TSDB_CODE_SUCCESS;
}
static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) {
SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
if (NULL == info) {
tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType);
return TSDB_CODE_SUCCESS;
......@@ -116,7 +114,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRs
int32_t kvNum = pRsp->info ? taosArrayGetSize(pRsp->info) : 0;
tscDebug("hb got %d rsp kv", kvNum);
for (int32_t i = 0; i < kvNum; ++i) {
SKv *kv = taosArrayGet(pRsp->info, i);
switch (kv->key) {
......@@ -126,30 +124,30 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRs
break;
}
int64_t *clusterId = (int64_t *)info->param;
int64_t *clusterId = (int64_t *)info->param;
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(*clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code));
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code));
break;
}
hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog);
break;
}
case HEARTBEAT_KEY_STBINFO:{
case HEARTBEAT_KEY_STBINFO: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
int64_t *clusterId = (int64_t *)info->param;
int64_t *clusterId = (int64_t *)info->param;
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(*clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code));
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code));
break;
}
......@@ -165,22 +163,22 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRs
return TSDB_CODE_SUCCESS;
}
static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code) {
static int32_t hbMqAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) {
static int32_t emptyRspNum = 0;
if (code != 0) {
tfree(param);
return -1;
}
char *key = (char *)param;
char *key = (char *)param;
SClientHbBatchRsp pRsp = {0};
tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp);
int32_t rspNum = taosArrayGetSize(pRsp.rsps);
SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
if (pInst == NULL || NULL == *pInst) {
tscError("cluster not exist, key:%s", key);
tscError("cluster not exist, key:%s", key);
tfree(param);
tFreeClientHbBatchRsp(&pRsp);
return -1;
......@@ -189,13 +187,14 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code
tfree(param);
if (rspNum) {
tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0));
tscDebug("hb got %d rsp, %d empty rsp received before", rspNum,
atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0));
} else {
atomic_add_fetch_32(&emptyRspNum, 1);
}
for (int32_t i = 0; i < rspNum; ++i) {
SClientHbRsp* rsp = taosArrayGet(pRsp.rsps, i);
SClientHbRsp *rsp = taosArrayGet(pRsp.rsps, i);
code = (*clientHbMgr.rspHandle[rsp->connKey.hbType])((*pInst)->pAppHbMgr, rsp);
if (code) {
break;
......@@ -203,14 +202,14 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code
}
tFreeClientHbBatchRsp(&pRsp);
return code;
}
int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) {
SDbVgVersion *dbs = NULL;
uint32_t dbNum = 0;
int32_t code = 0;
uint32_t dbNum = 0;
int32_t code = 0;
code = catalogGetExpiredDBs(pCatalog, &dbs, &dbNum);
if (TSDB_CODE_SUCCESS != code) {
......@@ -238,8 +237,8 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) {
SSTableMetaVersion *stbs = NULL;
uint32_t stbNum = 0;
int32_t code = 0;
uint32_t stbNum = 0;
int32_t code = 0;
code = catalogGetExpiredSTables(pCatalog, &stbs, &stbNum);
if (TSDB_CODE_SUCCESS != code) {
......@@ -254,7 +253,7 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
SSTableMetaVersion *stb = &stbs[i];
stb->suid = htobe64(stb->suid);
stb->sversion = htons(stb->sversion);
stb->tversion = htons(stb->tversion);
stb->tversion = htons(stb->tversion);
}
SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs};
......@@ -266,17 +265,16 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
return TSDB_CODE_SUCCESS;
}
int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req) {
int64_t *clusterId = (int64_t *)param;
int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {
int64_t *clusterId = (int64_t *)param;
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(*clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code));
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code));
return code;
}
code = hbGetExpiredDBInfo(connKey, pCatalog, req);
if (TSDB_CODE_SUCCESS != code) {
return code;
......@@ -287,13 +285,10 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req
return code;
}
return TSDB_CODE_SUCCESS;
}
int32_t hbMqHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req) {
}
int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {}
void hbMgrInitMqHbHandle() {
clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle;
......@@ -312,10 +307,8 @@ void hbFreeReq(void *req) {
tFreeReqKvHash(pReq->info);
}
SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
SClientHbBatchReq* pBatchReq = calloc(1, sizeof(SClientHbBatchReq));
SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
SClientHbBatchReq *pBatchReq = calloc(1, sizeof(SClientHbBatchReq));
if (pBatchReq == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return NULL;
......@@ -324,11 +317,11 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq));
int32_t code = 0;
void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL);
void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL);
while (pIter != NULL) {
SClientHbReq* pOneReq = pIter;
SClientHbReq *pOneReq = pIter;
SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey));
SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey));
if (info) {
code = (*clientHbMgr.reqHandle[pOneReq->connKey.hbType])(&pOneReq->connKey, info->param, pOneReq);
if (code) {
......@@ -350,11 +343,10 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
return pBatchReq;
}
void hbClearReqInfo(SAppHbMgr *pAppHbMgr) {
void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL);
while (pIter != NULL) {
SClientHbReq* pOneReq = pIter;
SClientHbReq *pOneReq = pIter;
tFreeReqKvHash(pOneReq->info);
taosHashClear(pOneReq->info);
......@@ -363,31 +355,29 @@ void hbClearReqInfo(SAppHbMgr *pAppHbMgr) {
}
}
static void* hbThreadFunc(void* param) {
static void *hbThreadFunc(void *param) {
setThreadName("hb");
while (1) {
int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2);
if(1 == threadStop) {
if (1 == threadStop) {
break;
}
pthread_mutex_lock(&clientHbMgr.lock);
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
for(int i = 0; i < sz; i++) {
SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
for (int i = 0; i < sz; i++) {
SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt);
if (connCnt == 0) {
continue;
}
SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr);
SClientHbBatchReq *pReq = hbGatherAllInfo(pAppHbMgr);
if (pReq == NULL) {
continue;
}
int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq);
int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq);
void *buf = malloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
......@@ -395,7 +385,7 @@ static void* hbThreadFunc(void* param) {
hbClearReqInfo(pAppHbMgr);
break;
}
tSerializeSClientHbBatchReq(buf, tlen, pReq);
SMsgSendInfo *pInfo = calloc(1, sizeof(SMsgSendInfo));
......@@ -415,17 +405,17 @@ static void* hbThreadFunc(void* param) {
pInfo->requestObjRefId = 0;
SAppInstInfo *pAppInstInfo = pAppHbMgr->pAppInstInfo;
int64_t transporterId = 0;
SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
int64_t transporterId = 0;
SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo);
tFreeClientHbBatchReq(pReq, false);
tFreeClientHbBatchReq(pReq, false);
hbClearReqInfo(pAppHbMgr);
atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1);
}
pthread_mutex_unlock(&clientHbMgr.lock);
taosMsleep(HEARTBEAT_INTERVAL);
}
return NULL;
......@@ -449,17 +439,18 @@ static void hbStopThread() {
tscDebug("hb thread already stopped");
return;
}
while (2 != atomic_load_8(&clientHbMgr.threadStop)) {
usleep(10);
}
tscDebug("hb thread stopped");
tscDebug("hb thread stopped");
}
SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) {
SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
/*return NULL;*/
hbMgrInit();
SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr));
SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr));
if (pAppHbMgr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
......@@ -496,7 +487,7 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) {
pthread_mutex_lock(&clientHbMgr.lock);
taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr);
pthread_mutex_unlock(&clientHbMgr.lock);
return pAppHbMgr;
}
......@@ -505,7 +496,7 @@ void appHbMgrCleanup(void) {
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
for (int i = 0; i < sz; i++) {
SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
taosHashCleanup(pTarget->activeInfo);
pTarget->activeInfo = NULL;
taosHashCleanup(pTarget->connInfo);
......@@ -516,11 +507,12 @@ void appHbMgrCleanup(void) {
}
int hbMgrInit() {
/*return 0;*/
// init once
int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1);
if (old == 1) return 0;
clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void*));
clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *));
pthread_mutex_init(&clientHbMgr.lock, NULL);
// init handle funcs
......@@ -535,34 +527,34 @@ int hbMgrInit() {
void hbMgrCleanUp() {
return;
hbStopThread();
// destroy all appHbMgr
int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0);
if (old == 0) return;
pthread_mutex_lock(&clientHbMgr.lock);
appHbMgrCleanup();
taosArrayDestroy(clientHbMgr.appHbMgrs);
taosArrayDestroy(clientHbMgr.appHbMgrs);
pthread_mutex_unlock(&clientHbMgr.lock);
clientHbMgr.appHbMgrs = NULL;
}
int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) {
int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) {
// init hash in activeinfo
void* data = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
void *data = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
if (data != NULL) {
return 0;
}
SClientHbReq hbReq;
hbReq.connKey = connKey;
hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq));
// init hash
if (info != NULL) {
SClientHbReq * pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
info->req = pReq;
taosHashPut(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey), info, sizeof(SHbConnInfo));
}
......@@ -571,9 +563,10 @@ int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo *
return 0;
}
int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) {
int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) {
/*return 0;*/
SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY};
SHbConnInfo info = {0};
SHbConnInfo info = {0};
switch (hbType) {
case HEARTBEAT_TYPE_QUERY: {
......@@ -589,11 +582,12 @@ int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int3
default:
break;
}
return hbRegisterConnImpl(pAppHbMgr, connKey, &info);
}
void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) {
void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
/*return;*/
int32_t code = 0;
code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
......@@ -603,9 +597,11 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) {
atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1);
}
int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) {
int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen,
int32_t valueLen) {
return 0;
// find req by connection id
SClientHbReq* pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
ASSERT(pReq != NULL);
taosHashPut(pReq->info, key, keyLen, value, valueLen);
......
source/client/src/clientImpl.c
浏览文件 @ f0e392ae
......@@ -227,10 +227,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) {
void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;
SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res);
int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, &res);
if (code != TSDB_CODE_SUCCESS) {
if (pRequest->body.pQueryJob != NULL) {
schedulerFreeJob(pRequest->body.pQueryJob);
if (pRequest->body.queryJob != 0) {
schedulerFreeJob(pRequest->body.queryJob);
}
pRequest->code = code;
......@@ -240,8 +240,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList)
if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
pRequest->body.resInfo.numOfRows = res.numOfRows;
if (pRequest->body.pQueryJob != NULL) {
schedulerFreeJob(pRequest->body.pQueryJob);
if (pRequest->body.queryJob != 0) {
schedulerFreeJob(pRequest->body.queryJob);
}
}
......@@ -494,7 +494,7 @@ void* doFetchRow(SRequestObj* pRequest) {
}
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
int32_t code = schedulerFetchRows(pRequest->body.pQueryJob, (void**)&pResInfo->pData);
int32_t code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData);
if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code;
return NULL;
......
source/client/src/clientMsgHandler.c
浏览文件 @ f0e392ae
......@@ -19,6 +19,7 @@
#include "clientInt.h"
#include "clientLog.h"
#include "catalog.h"
#include "query.h"
int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
......@@ -243,6 +244,23 @@ int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param;
if (TSDB_CODE_MND_DB_NOT_EXIST == code) {
SUseDbRsp usedbRsp = {0};
tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp);
struct SCatalog *pCatalog = NULL;
if (usedbRsp.vgVersion >= 0) {
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code));
} else {
catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid);
}
}
tFreeSUsedbRsp(&usedbRsp);
}
if (code != TSDB_CODE_SUCCESS) {
free(pMsg->pData);
setErrno(pRequest, code);
......@@ -256,6 +274,26 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SName name = {0};
tNameFromString(&name, usedbRsp.db, T_NAME_ACCT|T_NAME_DB);
SUseDbOutput output = {0};
code = queryBuildUseDbOutput(&output, &usedbRsp);
if (code != 0) {
terrno = code;
if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash);
tfree(output.dbVgroup);
tscError("failed to build use db output since %s", terrstr());
} else {
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code));
} else {
catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
}
}
tFreeSUsedbRsp(&usedbRsp);
char db[TSDB_DB_NAME_LEN] = {0};
......
source/client/src/tmq.c
浏览文件 @ f0e392ae
......@@ -25,6 +25,7 @@
#include "tglobal.h"
#include "tmsgtype.h"
#include "tpagedbuf.h"
#include "tqueue.h"
#include "tref.h"
struct tmq_list_t {
......@@ -58,22 +59,28 @@ struct tmq_t {
char groupId[256];
char clientId[256];
int8_t autoCommit;
SRWLatch lock;
int64_t consumerId;
int32_t epoch;
int32_t resetOffsetCfg;
int64_t status;
tsem_t rspSem;
STscObj* pTscObj;
tmq_commit_cb* commit_cb;
int32_t nextTopicIdx;
SArray* clientTopics; // SArray<SMqClientTopic>
STaosQueue* mqueue; // queue of tmq_message_t
STaosQall* qall;
// stat
int64_t pollCnt;
};
struct tmq_message_t {
SMqConsumeRsp rsp;
enum {
TMQ_VG_STATUS__IDLE = 0,
TMQ_VG_STATUS__WAIT,
};
enum {
TMQ_CONSUMER_STATUS__INIT = 0,
TMQ_CONSUMER_STATUS__READY,
};
typedef struct {
......@@ -83,6 +90,7 @@ typedef struct {
int64_t currentOffset;
// connection info
int32_t vgId;
int32_t vgStatus;
SEpSet epSet;
} SMqClientVg;
......@@ -104,15 +112,16 @@ typedef struct {
typedef struct {
tmq_t* tmq;
int32_t wait;
int32_t sync;
tsem_t rspSem;
} SMqAskEpCbParam;
typedef struct {
tmq_t* tmq;
SMqClientVg* pVg;
tmq_message_t** retMsg;
tsem_t rspSem;
} SMqConsumeCbParam;
tmq_t* tmq;
SMqClientVg* pVg;
int32_t epoch;
tsem_t rspSem;
} SMqPollCbParam;
typedef struct {
tmq_t* tmq;
......@@ -125,7 +134,7 @@ typedef struct {
tmq_conf_t* tmq_conf_new() {
tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t));
conf->auto_commit = false;
conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST;
conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
return conf;
}
......@@ -209,6 +218,22 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
return 0;
}
tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
if (*topics == NULL) {
*topics = tmq_list_new();
}
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i);
tmq_list_append(*topics, strdup(topic->topicName));
}
return TMQ_RESP_ERR__SUCCESS;
}
tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) {
tmq_list_t* lst = tmq_list_new();
return tmq_subscribe(tmq, lst);
}
tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
tmq_t* pTmq = calloc(sizeof(tmq_t), 1);
if (pTmq == NULL) {
......@@ -218,7 +243,6 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
pTmq->status = 0;
pTmq->pollCnt = 0;
pTmq->epoch = 0;
taosInitRWLatch(&pTmq->lock);
// set conf
strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId);
......@@ -226,9 +250,11 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
pTmq->commit_cb = conf->commit_cb;
pTmq->resetOffsetCfg = conf->resetOffset;
tsem_init(&pTmq->rspSem, 0, 0);
pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
pTmq->mqueue = taosOpenQueue();
pTmq->qall = taosAllocateQall();
return pTmq;
}
......@@ -290,7 +316,11 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
pParam->tmq = tmq;
tsem_init(&pParam->rspSem, 0, 0);
pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen};
pRequest->body.requestMsg = (SDataBuf){
.pData = buf,
.len = tlen,
.handle = NULL,
};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->param = pParam;
......@@ -365,10 +395,17 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
tscError("failed to malloc request");
}
SMqSubscribeCbParam param = {.rspErr = TMQ_RESP_ERR__SUCCESS, .tmq = tmq};
SMqSubscribeCbParam param = {
.rspErr = TMQ_RESP_ERR__SUCCESS,
.tmq = tmq,
};
tsem_init(&param.rspSem, 0, 0);
pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL};
pRequest->body.requestMsg = (SDataBuf){
.pData = buf,
.len = tlen,
.handle = NULL,
};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->param = &param;
......@@ -391,36 +428,6 @@ _return:
void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; }
SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) {
tmq_t* pTmq = (void*)param;
SArray* pArray = taosArrayInit(0, sizeof(SKv));
if (pArray == NULL) {
return NULL;
}
SKv kv = {0};
kv.key = HEARTBEAT_KEY_MQ_TMP;
SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg));
if (pMqHb == NULL) {
return pArray;
}
pMqHb->consumerId = connKey.connId;
SArray* clientTopics = pTmq->clientTopics;
int sz = taosArrayGetSize(clientTopics);
for (int i = 0; i < sz; i++) {
SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i);
/*if (pCTopic->vgId == -1) {*/
/*pMqHb->status = 1;*/
/*break;*/
/*}*/
}
kv.value = pMqHb;
kv.valueLen = sizeof(SMqHbMsg);
taosArrayPush(pArray, &kv);
return pArray;
}
TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
STscObj* pTscObj = (STscObj*)taos;
SRequestObj* pRequest = NULL;
......@@ -578,7 +585,7 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
static bool noPrintSchema;
char pBuf[128];
SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message;
SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
int32_t colNum = pRsp->schemas->nCols;
if (!noPrintSchema) {
printf("|");
......@@ -618,94 +625,125 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
}
int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param;
SMqClientVg* pVg = pParam->pVg;
printf("recv poll\n");
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
SMqClientVg* pVg = pParam->pVg;
tmq_t* tmq = pParam->tmq;
if (code != 0) {
printf("msg discard\n");
tsem_post(&pParam->rspSem);
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
return 0;
}
SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));
int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
if (msgEpoch < tmqEpoch) {
printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
return 0;
}
if (msgEpoch != tmqEpoch) {
printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
}
/*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/
tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
if (pRsp == NULL) {
tsem_post(&pParam->rspSem);
printf("fail\n");
return -1;
}
tDecodeSMqConsumeRsp(pMsg->pData, pRsp);
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
/*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
if (pRsp->numOfTopics == 0) {
/*printf("no data\n");*/
free(pRsp);
tsem_post(&pParam->rspSem);
if (pRsp->consumeRsp.numOfTopics == 0) {
printf("no data\n");
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
taosFreeQitem(pRsp);
return 0;
}
*pParam->retMsg = (tmq_message_t*)pRsp;
pVg->currentOffset = pRsp->rspOffset;
pRsp->extra = pParam->pVg;
taosWriteQitem(tmq->mqueue, pRsp);
printf("poll in queue\n");
/*pParam->rspMsg = (tmq_message_t*)pRsp;*/
/*pVg->currentOffset = pRsp->consumeRsp.rspOffset;*/
/*printf("rsp offset: %ld\n", rsp.rspOffset);*/
/*printf("-----msg begin----\n");*/
tsem_post(&pParam->rspSem);
/*printf("\n-----msg end------\n");*/
return 0;
}
bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
bool set = false;
int32_t sz = taosArrayGetSize(pRsp->topics);
if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics);
tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic));
for (int32_t i = 0; i < sz; i++) {
SMqClientTopic topic = {0};
SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i);
topic.topicName = strdup(pTopicEp->topic);
int32_t vgSz = taosArrayGetSize(pTopicEp->vgs);
topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg));
for (int32_t j = 0; j < vgSz; j++) {
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
SMqClientVg clientVg = {
.pollCnt = 0,
.currentOffset = pVgEp->offset,
.vgId = pVgEp->vgId,
.epSet = pVgEp->epSet,
.vgStatus = TMQ_VG_STATUS__IDLE,
};
taosArrayPush(topic.vgs, &clientVg);
set = true;
}
taosArrayPush(tmq->clientTopics, &topic);
}
atomic_store_32(&tmq->epoch, epoch);
return set;
}
int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
tmq_t* tmq = pParam->tmq;
if (code != 0) {
printf("get topic endpoint error, not ready, wait:%d\n", pParam->wait);
if (pParam->wait) {
tsem_post(&tmq->rspSem);
printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
if (pParam->sync) {
tsem_post(&pParam->rspSem);
}
return 0;
}
tscDebug("tmq ask ep cb called");
bool set = false;
SMqCMGetSubEpRsp rsp;
tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp);
int32_t sz = taosArrayGetSize(rsp.topics);
// TODO: lock
/*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
/*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
if (rsp.epoch != tmq->epoch) {
// TODO
if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics);
tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic));
for (int32_t i = 0; i < sz; i++) {
SMqClientTopic topic = {0};
SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i);
topic.topicName = strdup(pTopicEp->topic);
int32_t vgSz = taosArrayGetSize(pTopicEp->vgs);
topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg));
for (int32_t j = 0; j < vgSz; j++) {
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
// clang-format off
SMqClientVg clientVg = {
.pollCnt = 0,
.currentOffset = pVgEp->offset,
.vgId = pVgEp->vgId,
.epSet = pVgEp->epSet
};
// clang-format on
taosArrayPush(topic.vgs, &clientVg);
set = true;
}
taosArrayPush(tmq->clientTopics, &topic);
if (pParam->sync) {
SMqRspHead* head = pMsg->pData;
SMqCMGetSubEpRsp rsp;
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
/*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
/*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
int32_t epoch = atomic_load_32(&tmq->epoch);
if (head->epoch > epoch && tmqUpdateEp(tmq, head->epoch, &rsp)) {
atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY);
}
tmq->epoch = rsp.epoch;
}
if (set) {
atomic_store_64(&tmq->status, 1);
}
// unlock
/*tsem_post(&tmq->rspSem);*/
if (pParam->wait) {
tsem_post(&tmq->rspSem);
tsem_post(&pParam->rspSem);
tDeleteSMqCMGetSubEpRsp(&rsp);
} else {
tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp);
taosWriteQitem(tmq->mqueue, pRsp);
}
tDeleteSMqCMGetSubEpRsp(&rsp);
return 0;
}
int32_t tmqAskEp(tmq_t* tmq, bool wait) {
int32_t tmqAskEp(tmq_t* tmq, bool sync) {
printf("ask ep sync %d\n", sync);
int32_t tlen = sizeof(SMqCMGetSubEpReq);
SMqCMGetSubEpReq* buf = malloc(tlen);
if (buf == NULL) {
......@@ -722,7 +760,11 @@ int32_t tmqAskEp(tmq_t* tmq, bool wait) {
goto END;
}
pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen};
pRequest->body.requestMsg = (SDataBuf){
.pData = buf,
.len = tlen,
.handle = NULL,
};
SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam));
if (pParam == NULL) {
......@@ -730,7 +772,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool wait) {
goto END;
}
pParam->tmq = tmq;
pParam->wait = wait;
pParam->sync = sync;
tsem_init(&pParam->rspSem, 0, 0);
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
......@@ -743,7 +786,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool wait) {
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
END:
if (wait) tsem_wait(&tmq->rspSem);
if (sync) tsem_wait(&pParam->rspSem);
return 0;
}
......@@ -791,6 +834,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie
pReq->blockingTime = blocking_time;
pReq->consumerId = tmq->consumerId;
pReq->epoch = tmq->epoch;
pReq->currentOffset = reqOffset;
pReq->head.vgId = htonl(pVg->vgId);
......@@ -798,11 +842,158 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie
return pReq;
}
void tmqClearUnhandleMsg(tmq_t* tmq) {
tmq_message_t* msg;
while (1) {
taosGetQitem(tmq->qall, (void**)&msg);
if (msg)
taosFreeQitem(msg);
else
break;
}
taosReadAllQitems(tmq->mqueue, tmq->qall);
while (1) {
taosGetQitem(tmq->qall, (void**)&msg);
if (msg)
taosFreeQitem(msg);
else
break;
}
}
int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
printf("call poll\n");
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
if (vgStatus != TMQ_VG_STATUS__IDLE) {
continue;
}
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem
return -1;
}
SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam));
if (param == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem
return -1;
}
param->tmq = tmq;
param->pVg = pVg;
param->epoch = tmq->epoch;
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqConsumeReq),
.handle = NULL,
};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
sendInfo->param = param;
sendInfo->fp = tmqPollCb;
int64_t transporterId = 0;
printf("send poll\n");
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
pVg->pollCnt++;
tmq->pollCnt++;
}
}
return 0;
}
// return
int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) {
if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);
if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) {
tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp);
tmqClearUnhandleMsg(tmq);
*pReset = true;
} else {
*pReset = false;
}
} else {
return -1;
}
return 0;
}
tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
while (1) {
tmq_message_t* rspMsg = NULL;
taosGetQitem(tmq->qall, (void**)&rspMsg);
if (rspMsg == NULL) {
break;
}
if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) {
printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);
if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) {
printf("epoch match\n");
SMqClientVg* pVg = rspMsg->extra;
pVg->currentOffset = rspMsg->consumeRsp.rspOffset;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
return rspMsg;
} else {
printf("epoch mismatch\n");
taosFreeQitem(rspMsg);
}
} else {
printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);
bool reset = false;
tmqHandleRes(tmq, rspMsg, &reset);
taosFreeQitem(rspMsg);
if (pollIfReset && reset) {
printf("reset and repoll\n");
tmqPollImpl(tmq, blockingTime);
}
}
}
return NULL;
}
tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* tmq_message = NULL;
tmq_message_t* rspMsg = NULL;
int64_t startTime = taosGetTimestampMs();
// TODO: put into another thread or delayed queue
int64_t status = atomic_load_64(&tmq->status);
tmqAskEp(tmq, status == 0);
tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
taosGetQitem(tmq->qall, (void**)&rspMsg);
if (rspMsg == NULL) {
taosReadAllQitems(tmq->mqueue, tmq->qall);
}
tmqHandleAllRsp(tmq, blocking_time, false);
tmqPollImpl(tmq, blocking_time);
while (1) {
/*printf("cycle\n");*/
taosReadAllQitems(tmq->mqueue, tmq->qall);
rspMsg = tmqHandleAllRsp(tmq, blocking_time, true);
if (rspMsg) {
return rspMsg;
}
if (blocking_time != 0) {
int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) {
printf("normal exit\n");
return NULL;
}
}
}
}
#if 0
if (blocking_time <= 0) blocking_time = 1;
if (blocking_time > 1000) blocking_time = 1000;
......@@ -834,7 +1025,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
return NULL;
}
SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam));
SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam));
if (param == NULL) {
ASSERT(false);
usleep(blocking_time * 1000);
......@@ -846,7 +1037,11 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
tsem_init(&param->rspSem, 0, 0);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq), .handle = NULL};
pRequest->body.requestMsg = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqConsumeReq),
.handle = NULL,
};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
......@@ -886,6 +1081,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
/*return pRequest;*/
}
#endif
#if 0
tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) {
......@@ -929,9 +1125,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
void tmq_message_destroy(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return;
SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message;
SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
tDeleteSMqConsumeRsp(pRsp);
free(tmq_message);
taosFreeQitem(tmq_message);
}
tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; }
......
source/common/CMakeLists.txt
浏览文件 @ f0e392ae
aux_source_directory(src COMMON_SRC)
add_library(common ${COMMON_SRC})
add_library(common STATIC ${COMMON_SRC})
target_include_directories(
common
PUBLIC "${CMAKE_SOURCE_DIR}/include/common"
......
source/common/src/tglobal.c
浏览文件 @ f0e392ae
......@@ -46,6 +46,13 @@ int32_t tsMaxBinaryDisplayWidth = 30;
bool tsEnableSlaveQuery = 1;
bool tsPrintAuth = 0;
// monitor
bool tsEnableMonitor = 1;
int32_t tsMonitorInterval = 5;
char tsMonitorFqdn[TSDB_FQDN_LEN] = {0};
uint16_t tsMonitorPort = 6043;
int32_t tsMonitorMaxLogs = 100;
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
* metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
......@@ -177,24 +184,24 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr());
return -1;
}
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
uError("failed to load from config file:%s since %s\n", cfgFile, terrstr());
return -1;
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
uError("failed to load from config file:%s since %s", cfgFile, terrstr());
return 0;
}
}
if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) {
uError("failed to load from env file:%s since %s\n", envFile, terrstr());
uError("failed to load from env file:%s since %s", envFile, terrstr());
return -1;
}
if (cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL) != 0) {
uError("failed to load from global env variables since %s\n", terrstr());
uError("failed to load from global env variables since %s", terrstr());
return -1;
}
......@@ -271,7 +278,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddTimezone(pCfg, "timezone", tsTimezone) != 0) return -1;
if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1;
if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 0, 1) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
......@@ -314,6 +321,13 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1;
if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1;
if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1;
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1;
if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1;
return 0;
}
......@@ -345,7 +359,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
}
static void taosSetClientCfg(SConfig *pCfg) {
tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_EP_LEN);
tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN);
tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
......@@ -425,6 +439,12 @@ static void taosSetServerCfg(SConfig *pCfg) {
tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval;
tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN);
tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32;
tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
}
......@@ -438,8 +458,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
if (pCfg == NULL) return -1;
if (tsc) {
tscEmbeddedInUtil = 0;
if (taosAddClientLogCfg(pCfg) != 0) return -1;
} else {
tscEmbeddedInUtil = 1;
if (taosAddClientLogCfg(pCfg) != 0) return -1;
if (taosAddServerLogCfg(pCfg) != 0) return -1;
}
......@@ -450,7 +472,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
return -1;
}
if (cfgLoadArray(pCfg, pArgs) != 0) {
if (cfgLoadFromArray(pCfg, pArgs) != 0) {
uError("failed to load cfg from array since %s", terrstr());
cfgCleanup(pCfg);
return -1;
......@@ -465,8 +487,14 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32);
if (taosMkDir(tsLogDir) != 0) {
uError("failed to create dir:%s since %s", tsLogDir, terrstr());
cfgCleanup(pCfg);
return -1;
}
if (taosInitLog(logname, logFileNum) != 0) {
printf("failed to init log file since %s\n", terrstr());
uError("failed to init log file since %s", terrstr());
cfgCleanup(pCfg);
return -1;
}
......@@ -497,7 +525,7 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
return -1;
}
if (cfgLoadArray(tsCfg, pArgs) != 0) {
if (cfgLoadFromArray(tsCfg, pArgs) != 0) {
uError("failed to load cfg from array since %s", terrstr());
cfgCleanup(tsCfg);
return -1;
......@@ -512,6 +540,16 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
}
taosSetSystemCfg(tsCfg);
if (taosMkDir(tsTempDir) != 0) {
uError("failed to create dir:%s since %s", tsTempDir, terrstr());
return -1;
}
if (!tsc && taosMkDir(tsDataDir) != 0) {
uError("failed to create dir:%s since %s", tsDataDir, terrstr());
return -1;
}
cfgDumpCfg(tsCfg, tsc, false);
return 0;
}
......
source/common/src/tmsg.c
浏览文件 @ f0e392ae
......@@ -270,7 +270,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
int32_t rspNum = 0;
if (tDecodeI32(&decoder, &rspNum) < 0) return -1;
if (pBatchRsp->rsps == NULL) {
pBatchRsp->rsps = taosArrayInit(rspNum, sizeof(SClientHbReq));
pBatchRsp->rsps = taosArrayInit(rspNum, sizeof(SClientHbRsp));
}
for (int32_t i = 0; i < rspNum; i++) {
SClientHbRsp rsp = {0};
......@@ -1421,6 +1421,7 @@ int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) {
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
tEndEncode(&encoder);
......@@ -1435,6 +1436,7 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) {
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
tEndDecode(&decoder);
......@@ -1527,7 +1529,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
if (pRsp->vgNum <= 0) {
return 0;
}
pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo));
if (pRsp->pVgroupInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
......@@ -2110,6 +2112,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1;
if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1;
if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
if (tEncodeU32(&encoder, pReq->hashBegin) < 0) return -1;
if (tEncodeU32(&encoder, pReq->hashEnd) < 0) return -1;
if (tEncodeI8(&encoder, pReq->hashMethod) < 0) return -1;
if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
......@@ -2150,6 +2155,9 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
if (tDecodeU32(&decoder, &pReq->hashBegin) < 0) return -1;
if (tDecodeU32(&decoder, &pReq->hashEnd) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->hashMethod) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
......
source/common/src/tname.c
浏览文件 @ f0e392ae
......@@ -22,6 +22,7 @@ bool tscValidateTableNameLength(size_t len) {
return len < TSDB_TABLE_NAME_LEN;
}
#if 0
// TODO refactor
SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters) {
if (numOfFilters == 0 || src == NULL) {
......@@ -46,7 +47,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil
return pFilter;
}
#endif
#if 0
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision) {
if (slidingTime == 0) {
......
source/common/src/ttime.c
浏览文件 @ f0e392ae
......@@ -627,3 +627,50 @@ const char* fmtts(int64_t ts) {
return buf;
}
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
char ts[40] = {0};
struct tm* ptm;
int32_t fractionLen;
char* format = NULL;
time_t quot = 0;
long mod = 0;
switch (precision) {
case TSDB_TIME_PRECISION_MILLI: {
quot = t / 1000;
fractionLen = 5;
format = ".%03" PRId64;
mod = t % 1000;
break;
}
case TSDB_TIME_PRECISION_MICRO: {
quot = t / 1000000;
fractionLen = 8;
format = ".%06" PRId64;
mod = t % 1000000;
break;
}
case TSDB_TIME_PRECISION_NANO: {
quot = t / 1000000000;
fractionLen = 11;
format = ".%09" PRId64;
mod = t % 1000000000;
break;
}
default:
fractionLen = 0;
assert(false);
}
ptm = localtime(&quot);
int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm);
length += snprintf(ts + length, fractionLen, format, mod);
length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm);
tstrncpy(buf, ts, bufLen);
}
\ No newline at end of file
source/common/src/ttypes.c
浏览文件 @ f0e392ae
......@@ -49,16 +49,16 @@ const int32_t TYPE_BYTES[15] = {
} \
} while (0)
static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
int8_t *data = (int8_t *)pData;
*min = INT64_MAX;
*max = INT64_MIN;
*minIndex = 0;
*maxIndex = 0;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if (data[i] == TSDB_DATA_BOOL_NULL) {
(*numOfNull) += 1;
......@@ -76,9 +76,9 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int
*max = INT64_MIN;
*minIndex = 0;
*maxIndex = 0;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) {
(*numOfNull) += 1;
......@@ -122,9 +122,9 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in
*max = INT64_MIN;
*minIndex = 0;
*maxIndex = 0;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) {
(*numOfNull) += 1;
......@@ -133,15 +133,14 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in
DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i);
}
}
static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
uint16_t *data = (uint16_t *)pData;
uint64_t _min = UINT64_MAX;
uint64_t _max = 0;
uint64_t _sum = 0;
uint64_t _min = UINT64_MAX;
uint64_t _max = 0;
uint64_t _sum = 0;
*minIndex = 0;
*maxIndex = 0;
......@@ -169,9 +168,9 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in
*max = INT64_MIN;
*minIndex = 0;
*maxIndex = 0;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) {
(*numOfNull) += 1;
......@@ -182,12 +181,12 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in
}
}
static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
uint32_t *data = (uint32_t *)pData;
uint64_t _min = UINT64_MAX;
uint64_t _max = 0;
uint64_t _sum = 0;
uint64_t _min = UINT64_MAX;
uint64_t _max = 0;
uint64_t _sum = 0;
*minIndex = 0;
*maxIndex = 0;
......@@ -208,16 +207,16 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in
*sum = _sum;
}
static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
int64_t *data = (int64_t *)pData;
*min = INT64_MAX;
*max = INT64_MIN;
*minIndex = 0;
*maxIndex = 0;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) {
(*numOfNull) += 1;
......@@ -228,12 +227,12 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in
}
}
static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
uint64_t *data = (uint64_t *)pData;
uint64_t _min = UINT64_MAX;
uint64_t _max = 0;
uint64_t _sum = 0;
uint64_t _min = UINT64_MAX;
uint64_t _max = 0;
uint64_t _sum = 0;
*minIndex = 0;
*maxIndex = 0;
......@@ -254,91 +253,91 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in
*sum = _sum;
}
static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
float *data = (float *)pData;
float fmin = FLT_MAX;
float fmax = -FLT_MAX;
double dsum = 0;
*minIndex = 0;
*maxIndex = 0;
static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
float *data = (float *)pData;
float fmin = FLT_MAX;
float fmax = -FLT_MAX;
double dsum = 0;
*minIndex = 0;
*maxIndex = 0;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if ((*(uint32_t*)&(data[i])) == TSDB_DATA_FLOAT_NULL) {
if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) {
(*numOfNull) += 1;
continue;
}
float fv = GET_FLOAT_VAL((const char*)&(data[i]));
float fv = GET_FLOAT_VAL((const char *)&(data[i]));
dsum += fv;
if (fmin > fv) {
fmin = fv;
*minIndex = i;
}
if (fmax < fv) {
fmax = fv;
*maxIndex = i;
}
}
SET_DOUBLE_VAL(sum, dsum);
SET_DOUBLE_VAL(max, fmax);
SET_DOUBLE_VAL(min, fmin);
}
static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
double *data = (double *)pData;
double dmin = DBL_MAX;
double dmax = -DBL_MAX;
double dsum = 0;
*minIndex = 0;
*maxIndex = 0;
double dmin = DBL_MAX;
double dmax = -DBL_MAX;
double dsum = 0;
*minIndex = 0;
*maxIndex = 0;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if ((*(uint64_t*)&(data[i])) == TSDB_DATA_DOUBLE_NULL) {
if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) {
(*numOfNull) += 1;
continue;
}
double dv = 0;
dv = GET_DOUBLE_VAL((const char*)&(data[i]));
dv = GET_DOUBLE_VAL((const char *)&(data[i]));
dsum += dv;
if (dmin > dv) {
dmin = dv;
*minIndex = i;
}
if (dmax < dv) {
dmax = dv;
*maxIndex = i;
}
}
SET_DOUBLE_PTR(sum, &dsum);
SET_DOUBLE_PTR(max, &dmax);
SET_DOUBLE_PTR(min, &dmin);
}
static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
const char* data = pData;
static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
const char *data = pData;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if (isNull(data, TSDB_DATA_TYPE_BINARY)) {
(*numOfNull) += 1;
}
data += varDataTLen(data);
}
*sum = 0;
*max = 0;
*min = 0;
......@@ -346,19 +345,19 @@ static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, in
*maxIndex = 0;
}
static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
const char* data = pData;
static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
const char *data = pData;
assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) {
if (isNull(data, TSDB_DATA_TYPE_NCHAR)) {
(*numOfNull) += 1;
}
data += varDataTLen(data);
}
*sum = 0;
*max = 0;
*min = 0;
......@@ -367,21 +366,28 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i
}
tDataTypeDescriptor tDataTypes[15] = {
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_i16},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32},
{TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_i64},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
{TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8},
{TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16},
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64},
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint,
getStatics_i8},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint,
tsDecompressSmallint, getStatics_i16},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32},
{TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint,
getStatics_i64},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp,
tsDecompressTimestamp, getStatics_i64},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
{TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint,
getStatics_u8},
{TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint,
tsDecompressSmallint, getStatics_u16},
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint,
getStatics_u64},
};
char tTokenTypeSwitcher[13] = {
......@@ -400,10 +406,10 @@ char tTokenTypeSwitcher[13] = {
TSDB_DATA_TYPE_NCHAR, // TK_NCHAR
};
float floatMin = -FLT_MAX, floatMax = FLT_MAX;
float floatMin = -FLT_MAX, floatMax = FLT_MAX;
double doubleMin = -DBL_MAX, doubleMax = DBL_MAX;
FORCE_INLINE void* getDataMin(int32_t type) {
FORCE_INLINE void *getDataMin(int32_t type) {
switch (type) {
case TSDB_DATA_TYPE_FLOAT:
return &floatMin;
......@@ -414,7 +420,7 @@ FORCE_INLINE void* getDataMin(int32_t type) {
}
}
FORCE_INLINE void* getDataMax(int32_t type) {
FORCE_INLINE void *getDataMax(int32_t type) {
switch (type) {
case TSDB_DATA_TYPE_FLOAT:
return &floatMax;
......@@ -425,17 +431,15 @@ FORCE_INLINE void* getDataMax(int32_t type) {
}
}
bool isValidDataType(int32_t type) {
return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT;
}
bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; }
void setVardataNull(void* val, int32_t type) {
void setVardataNull(void *val, int32_t type) {
if (type == TSDB_DATA_TYPE_BINARY) {
varDataSetLen(val, sizeof(int8_t));
*(uint8_t*) varDataVal(val) = TSDB_DATA_BINARY_NULL;
*(uint8_t *)varDataVal(val) = TSDB_DATA_BINARY_NULL;
} else if (type == TSDB_DATA_TYPE_NCHAR) {
varDataSetLen(val, sizeof(int32_t));
*(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL;
*(uint32_t *)varDataVal(val) = TSDB_DATA_NCHAR_NULL;
} else {
assert(0);
}
......@@ -533,9 +537,8 @@ static SBinaryNullT nullBinary = {1, TSDB_DATA_BINARY_NULL};
static SNCharNullT nullNchar = {4, TSDB_DATA_NCHAR_NULL};
static const void *nullValues[] = {
&nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt,
&nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar,
&nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu,
&nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, &nullFloat, &nullDouble,
&nullBinary, &nullBigInt, &nullNchar, &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu,
};
const void *getNullValue(int32_t type) {
......@@ -590,7 +593,7 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
*((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2);
break;
case TSDB_DATA_TYPE_UTINYINT:
*((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2);
*((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2);
break;
case TSDB_DATA_TYPE_SMALLINT:
*((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2);
......@@ -629,14 +632,14 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
}
}
void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) {
void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf) {
switch (type) {
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT: {
TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t);
break;
}
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
......@@ -652,19 +655,19 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
TSWAP(*(float *)(pLeft), *(float *)(pRight), float);
break;
}
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT: {
TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t);
break;
}
default: {
memcpy(buf, pLeft, size);
memcpy(pLeft, pRight, size);
......
source/common/src/tvariant.c
浏览文件 @ f0e392ae
......@@ -35,9 +35,9 @@
assert(0); \
} while (0)
int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* isSigned) {
int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value, bool *isSigned) {
errno = 0;
char* endPtr = NULL;
char *endPtr = NULL;
int32_t index = 0;
......@@ -63,15 +63,15 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool*
*isSigned = specifiedSign || (val <= INT64_MAX);
if (*isSigned) {
*value = (z[0] == '-')? -val:val;
*value = (z[0] == '-') ? -val : val;
} else {
*(uint64_t*) value = val;
*(uint64_t *)value = val;
}
return 0;
}
void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) {
void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) {
int32_t ret = 0;
memset(pVar, 0, sizeof(SVariant));
......@@ -90,7 +90,7 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) {
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_INT:{
case TSDB_DATA_TYPE_INT: {
bool sign = true;
int32_t base = 10;
......@@ -104,11 +104,11 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) {
ret = toInteger(z, n, base, &pVar->i, &sign);
if (ret != 0) {
pVar->nType = -1; // -1 means error type
pVar->nType = -1; // -1 means error type
return;
}
pVar->nType = (sign)? TSDB_DATA_TYPE_BIGINT:TSDB_DATA_TYPE_UBIGINT;
pVar->nType = (sign) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_UBIGINT;
break;
}
case TSDB_DATA_TYPE_DOUBLE:
......@@ -123,15 +123,15 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) {
}
case TSDB_DATA_TYPE_TIMESTAMP: {
assert(0);
pVar->i = taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
break;
}
pVar->i = taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
break;
}
default: { // nType == 0 means the null value
type = TSDB_DATA_TYPE_NULL;
}
}
pVar->nType = type;
}
......@@ -196,13 +196,13 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin
pVar->d = GET_FLOAT_VAL(pz);
break;
}
case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length
case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length
size_t lenInwchar = len / TSDB_NCHAR_SIZE;
pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE);
memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE);
pVar->nLen = (int32_t)len;
break;
}
case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method
......@@ -211,18 +211,18 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin
pVar->nLen = (int32_t)len;
break;
}
default:
pVar->i = GET_INT32_VAL(pz);
pVar->nLen = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
}
pVar->nType = type;
}
void taosVariantDestroy(SVariant *pVar) {
if (pVar == NULL) return;
if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) {
tfree(pVar->pz);
pVar->nLen = 0;
......@@ -231,8 +231,8 @@ void taosVariantDestroy(SVariant *pVar) {
// NOTE: this is only for string array
if (pVar->nType == TSDB_DATA_TYPE_POINTER_ARRAY) {
size_t num = taosArrayGetSize(pVar->arr);
for(size_t i = 0; i < num; i++) {
void* p = taosArrayGetP(pVar->arr, i);
for (size_t i = 0; i < num; i++) {
void *p = taosArrayGetP(pVar->arr, i);
free(p);
}
taosArrayDestroy(pVar->arr);
......@@ -250,11 +250,11 @@ bool taosVariantIsValid(SVariant *pVar) {
void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
if (pSrc == NULL || pDst == NULL) return;
pDst->nType = pSrc->nType;
if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
char* p = realloc(pDst->pz, len);
char *p = realloc(pDst->pz, len);
assert(p);
memset(p, 0, len);
......@@ -263,28 +263,27 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
memcpy(pDst->pz, pSrc->pz, pSrc->nLen);
pDst->nLen = pSrc->nLen;
return;
}
if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
pDst->i = pSrc->i;
} else if (pSrc->nType == TSDB_DATA_TYPE_POINTER_ARRAY) { // this is only for string array
size_t num = taosArrayGetSize(pSrc->arr);
pDst->arr = taosArrayInit(num, sizeof(char*));
for(size_t i = 0; i < num; i++) {
char* p = (char*)taosArrayGetP(pSrc->arr, i);
char* n = strdup(p);
pDst->arr = taosArrayInit(num, sizeof(char *));
for (size_t i = 0; i < num; i++) {
char *p = (char *)taosArrayGetP(pSrc->arr, i);
char *n = strdup(p);
taosArrayPush(pDst->arr, &n);
}
} else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
size_t num = taosArrayGetSize(pSrc->arr);
pDst->arr = taosArrayInit(num, sizeof(int64_t));
pDst->nLen = pSrc->nLen;
assert(pSrc->nLen == num);
for(size_t i = 0; i < num; i++) {
int64_t *p = taosArrayGet(pSrc->arr, i);
taosArrayPush(pDst->arr, p);
}
size_t num = taosArrayGetSize(pSrc->arr);
pDst->arr = taosArrayInit(num, sizeof(int64_t));
pDst->nLen = pSrc->nLen;
assert(pSrc->nLen == num);
for (size_t i = 0; i < num; i++) {
int64_t *p = taosArrayGet(pSrc->arr, i);
taosArrayPush(pDst->arr, p);
}
}
if (pDst->nType != TSDB_DATA_TYPE_POINTER_ARRAY && pDst->nType != TSDB_DATA_TYPE_VALUE_ARRAY) {
......@@ -292,7 +291,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
}
}
int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2) {
int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) {
if (p1->nType == TSDB_DATA_TYPE_NULL && p2->nType == TSDB_DATA_TYPE_NULL) {
return 0;
}
......@@ -309,39 +308,39 @@ int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2) {
if (p1->nLen == p2->nLen) {
return memcmp(p1->pz, p2->pz, p1->nLen);
} else {
return p1->nLen > p2->nLen? 1:-1;
return p1->nLen > p2->nLen ? 1 : -1;
}
} else if (p1->nType == TSDB_DATA_TYPE_FLOAT || p1->nType == TSDB_DATA_TYPE_DOUBLE) {
if (p1->d == p2->d) {
return 0;
} else {
return p1->d > p2->d? 1:-1;
return p1->d > p2->d ? 1 : -1;
}
} else if (IS_UNSIGNED_NUMERIC_TYPE(p1->nType)) {
if (p1->u == p2->u) {
return 0;
} else {
return p1->u > p2->u? 1:-1;
return p1->u > p2->u ? 1 : -1;
}
} else {
if (p1->i == p2->i) {
return 0;
} else {
return p1->i > p2->i? 1:-1;
return p1->i > p2->i ? 1 : -1;
}
}
}
int32_t taosVariantToString(SVariant *pVar, char *dst) {
if (pVar == NULL || dst == NULL) return 0;
switch (pVar->nType) {
case TSDB_DATA_TYPE_BINARY: {
int32_t len = sprintf(dst, "\'%s\'", pVar->pz);
assert(len <= pVar->nLen + sizeof("\'") * 2); // two more chars
return len;
}
case TSDB_DATA_TYPE_NCHAR: {
dst[0] = '\'';
taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
......@@ -350,7 +349,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) {
dst[len + 1] = 0;
return len + 1;
}
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
......@@ -359,7 +358,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) {
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
return sprintf(dst, "%d", (int32_t)pVar->i);
case TSDB_DATA_TYPE_BIGINT:
return sprintf(dst, "%" PRId64, pVar->i);
case TSDB_DATA_TYPE_UBIGINT:
......@@ -367,7 +366,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) {
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
return sprintf(dst, "%.9lf", pVar->d);
default:
return 0;
}
......@@ -399,27 +398,27 @@ static FORCE_INLINE int32_t wcsconvertToBoolImpl(wchar_t *pstr, int32_t len) {
static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
const int32_t INITIAL_ALLOC_SIZE = 40;
char * pBuf = NULL;
char *pBuf = NULL;
// it is a in-place convert type for SVariant, local buffer is needed
if (*pDest == pVariant->pz) {
pBuf = calloc(1, INITIAL_ALLOC_SIZE);
}
if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
size_t newSize = pVariant->nLen * TSDB_NCHAR_SIZE;
if (pBuf != NULL) {
if (newSize >= INITIAL_ALLOC_SIZE) {
pBuf = realloc(pBuf, newSize + 1);
}
taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf);
free(pVariant->wpz);
pBuf[newSize] = 0;
} else {
taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest);
}
} else {
if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) {
sprintf(pBuf == NULL ? *pDest : pBuf, "%" PRId64, pVariant->i);
......@@ -431,26 +430,26 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
setNull(pBuf == NULL ? *pDest : pBuf, TSDB_DATA_TYPE_BINARY, 0);
}
}
if (pBuf != NULL) {
*pDest = pBuf;
}
*pDestSize = (int32_t)strlen(*pDest);
return 0;
}
static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
char tmpBuf[40] = {0};
char * pDst = tmpBuf;
char *pDst = tmpBuf;
int32_t nLen = 0;
// convert the number to string, than convert it to wchar string.
if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) {
nLen = sprintf(pDst, "%" PRId64, pVariant->i);
} else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
nLen = sprintf(pDst, "%"PRIu64, pVariant->u);
nLen = sprintf(pDst, "%" PRIu64, pVariant->u);
} else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) {
nLen = sprintf(pDst, "%lf", pVariant->d);
} else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
......@@ -459,10 +458,10 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
} else if (pVariant->nType == TSDB_DATA_TYPE_BOOL) {
nLen = sprintf(pDst, "%s", (pVariant->i == TSDB_TRUE) ? "TRUE" : "FALSE");
}
if (*pDest == pVariant->pz) {
wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE);
bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
if (!ret) {
tfree(pWStr);
return -1;
......@@ -472,14 +471,14 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
free(pVariant->wpz);
}
pVariant->wpz = pWStr;
*pDestSize = twcslen(pVariant->wpz);
// shrink the allocate memory, no need to check here.
char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE);
char *tmp = realloc(pVariant->wpz, (*pDestSize + 1) * TSDB_NCHAR_SIZE);
assert(tmp != NULL);
pVariant->wpz = (wchar_t *)tmp;
} else {
int32_t output = 0;
......@@ -493,21 +492,22 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
*pDestSize = output;
}
}
return 0;
}
static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) {
// SToken stoken = {.z = pStr, .n = len};
// if (TK_ILLEGAL == tGetNumericStringType(&stoken)) {
// return -1;
// }
//
// *value = strtod(pStr, NULL);
// SToken stoken = {.z = pStr, .n = len};
// if (TK_ILLEGAL == tGetNumericStringType(&stoken)) {
// return -1;
// }
//
// *value = strtod(pStr, NULL);
return 0;
}
static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr, bool *converted) {
static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result, int32_t type, bool issigned,
bool releaseVariantPtr, bool *converted) {
if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
setNull((char *)result, type, tDataTypes[type].bytes);
return 0;
......@@ -645,7 +645,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) {
if ((ret = convertToBoolImpl(pVariant->pz, pVariant->nLen)) < 0) {
return ret;
}
*pDest = ret;
} else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
int32_t ret = 0;
......@@ -656,7 +656,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) {
} else if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
*pDest = TSDB_DATA_BOOL_NULL;
}
assert(*pDest == TSDB_TRUE || *pDest == TSDB_FALSE || *pDest == TSDB_DATA_BOOL_NULL);
return 0;
}
......@@ -665,11 +665,12 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) {
* transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType
* to column type defined in schema
*/
int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo) {
int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted,
char *extInfo) {
if (converted) {
*converted = false;
}
if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) {
return -1;
}
......@@ -686,25 +687,25 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
*(int8_t *)payload = (int8_t)result;
break;
}
case TSDB_DATA_TYPE_TINYINT: {
if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
SET_EXT_INFO(converted, result, INT8_MIN + 1, INT8_MAX, extInfo);
return -1;
}
*((int8_t *)payload) = (int8_t) result;
*((int8_t *)payload) = (int8_t)result;
break;
}
case TSDB_DATA_TYPE_UTINYINT: {
if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
SET_EXT_INFO(converted, result, 0, UINT8_MAX - 1, extInfo);
return -1;
}
*((uint8_t *)payload) = (uint8_t) result;
*((uint8_t *)payload) = (uint8_t)result;
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
SET_EXT_INFO(converted, result, INT16_MIN + 1, INT16_MAX, extInfo);
......@@ -722,7 +723,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
*((uint16_t *)payload) = (uint16_t)result;
break;
}
case TSDB_DATA_TYPE_INT: {
if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
SET_EXT_INFO(converted, result, INT32_MIN + 1, INT32_MAX, extInfo);
......@@ -740,7 +741,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
*((uint32_t *)payload) = (uint32_t)result;
break;
}
case TSDB_DATA_TYPE_BIGINT: {
if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo);
......@@ -775,19 +776,20 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
if (converted) {
*converted = true;
}
if (value > FLT_MAX || value < -FLT_MAX) {
SET_EXT_INFO(converted, value, -FLT_MAX, FLT_MAX, extInfo);
return -1;
}
SET_FLOAT_VAL(payload, value);
}
} else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
} else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) ||
IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
if (converted) {
*converted = true;
}
if (pVariant->i > FLT_MAX || pVariant->i < -FLT_MAX) {
if (pVariant->i > FLT_MAX || pVariant->i < -FLT_MAX) {
SET_EXT_INFO(converted, pVariant->i, -FLT_MAX, FLT_MAX, extInfo);
return -1;
}
......@@ -797,12 +799,12 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
if (converted) {
*converted = true;
}
if (pVariant->d > FLT_MAX || pVariant->d < -FLT_MAX) {
if (pVariant->d > FLT_MAX || pVariant->d < -FLT_MAX) {
SET_EXT_INFO(converted, pVariant->d, -FLT_MAX, FLT_MAX, extInfo);
return -1;
}
SET_FLOAT_VAL(payload, pVariant->d);
} else if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
*((uint32_t *)payload) = TSDB_DATA_FLOAT_NULL;
......@@ -831,7 +833,8 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
SET_DOUBLE_VAL(payload, value);
}
} else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
} else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) ||
IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
SET_DOUBLE_VAL(payload, pVariant->i);
} else if (IS_FLOAT_TYPE(pVariant->nType)) {
SET_DOUBLE_VAL(payload, pVariant->d);
......@@ -847,11 +850,11 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
break;
}
case TSDB_DATA_TYPE_BINARY: {
if (!includeLengthPrefix) {
if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
*(uint8_t*) payload = TSDB_DATA_BINARY_NULL;
*(uint8_t *)payload = TSDB_DATA_BINARY_NULL;
} else {
if (pVariant->nType != TSDB_DATA_TYPE_BINARY) {
toBinary(pVariant, &payload, &pVariant->nLen);
......@@ -918,11 +921,11 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
assert(p == varDataVal(payload));
}
}
break;
}
}
return 0;
}
......@@ -944,13 +947,13 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
if (pVariant == NULL || pVariant->nType == 0) { // value is not set
return 0;
}
switch (type) {
case TSDB_DATA_TYPE_BOOL: { // bool
if (convertToBool(pVariant, &pVariant->i) < 0) {
return -1;
}
pVariant->nType = type;
break;
}
......@@ -971,7 +974,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
free(pVariant->pz);
return -1;
}
free(pVariant->pz);
pVariant->d = v;
} else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
......@@ -981,14 +984,14 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
free(pVariant->pz);
return -1;
}
free(pVariant->pz);
pVariant->d = v;
} else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) {
double tmp = (double) pVariant->i;
double tmp = (double)pVariant->i;
pVariant->d = tmp;
}
pVariant->nType = TSDB_DATA_TYPE_DOUBLE;
break;
}
......@@ -1009,6 +1012,6 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
break;
}
}
return 0;
}
\ No newline at end of file
source/dnode/bnode/CMakeLists.txt
浏览文件 @ f0e392ae
aux_source_directory(src BNODE_SRC)
add_library(bnode ${BNODE_SRC})
add_library(bnode STATIC ${BNODE_SRC})
target_include_directories(
bnode
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/bnode"
......
source/dnode/mgmt/daemon/CMakeLists.txt
浏览文件 @ f0e392ae
......@@ -6,4 +6,4 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(taosd dnode util os)
target_link_libraries(taosd dnode)
source/dnode/mgmt/impl/CMakeLists.txt
浏览文件 @ f0e392ae
aux_source_directory(src DNODE_SRC)
add_library(dnode STATIC ${DNODE_SRC})
target_link_libraries(
dnode
PUBLIC cjson
PUBLIC mnode
PUBLIC vnode
PUBLIC qnode
PUBLIC snode
PUBLIC bnode
PUBLIC wal
PUBLIC sync
PUBLIC taos
PUBLIC tfs
dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
)
target_include_directories(
dnode
......
source/dnode/mgmt/impl/inc/dndEnv.h
浏览文件 @ f0e392ae
......@@ -137,6 +137,9 @@ typedef struct SDnode {
SStartupReq startup;
} SDnode;
int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo);
#ifdef __cplusplus
}
#endif
......
source/dnode/mgmt/impl/inc/dndInt.h
浏览文件 @ f0e392ae
......@@ -36,6 +36,7 @@ extern "C" {
#include "ttime.h"
#include "tworker.h"
#include "tglobal.h"
#include "monitor.h"
#include "dnode.h"
......
source/dnode/mgmt/impl/inc/dndMnode.h
浏览文件 @ f0e392ae
......@@ -32,6 +32,9 @@ int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo);
#ifdef __cplusplus
}
#endif
......
source/dnode/mgmt/impl/src/dndEnv.c
浏览文件 @ f0e392ae
......@@ -21,6 +21,7 @@
#include "dndSnode.h"
#include "dndTransport.h"
#include "dndVnodes.h"
#include "monitor.h"
#include "sync.h"
#include "tfs.h"
#include "wal.h"
......@@ -140,7 +141,7 @@ static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) {
return 0;
}
static void dndCloseImp(SDnode *pDnode) {
static void dndCloseDir(SDnode *pDnode) {
tfree(pDnode->dir.mnode);
tfree(pDnode->dir.vnodes);
tfree(pDnode->dir.dnode);
......@@ -260,7 +261,7 @@ void dndClose(SDnode *pDnode) {
dndCleanupMgmt(pDnode);
tfsClose(pDnode->pTfs);
dndCloseImp(pDnode);
dndCloseDir(pDnode);
free(pDnode);
dInfo("dnode object is closed, data:%p", pDnode);
}
......@@ -289,11 +290,7 @@ int32_t dndInit() {
}
SVnodeOpt vnodeOpt = {
.sver = tsVersion,
.nthreads = tsNumOfCommitThreads,
.putReqToVQueryQFp = dndPutReqToVQueryQ,
.sendReqToDnodeFp = dndSendReqToDnode
};
.nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode};
if (vnodeInit(&vnodeOpt) != 0) {
dError("failed to init vnode since %s", terrstr());
......@@ -301,6 +298,13 @@ int32_t dndInit() {
return -1;
}
SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn};
if (monInit(&monCfg) != 0) {
dError("failed to init monitor since %s", terrstr());
dndCleanup();
return -1;
}
dInfo("dnode env is initialized");
return 0;
}
......@@ -314,19 +318,10 @@ void dndCleanup() {
walCleanUp();
vnodeCleanup();
rpcCleanup();
monCleanup();
taosStopCacheRefreshWorker();
dInfo("dnode env is cleaned up");
}
// OTHER FUNCTIONS ===================================
void taosGetDisk() {
#if 0
const double unit = 1024 * 1024 * 1024;
SDiskSize diskSize = tfsGetSize(pTfs);
tfsUpdateSize(&fsMeta);
#endif
}
\ No newline at end of file
int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { return 0; }
\ No newline at end of file
source/dnode/mgmt/impl/src/dndMgmt.c
浏览文件 @ f0e392ae
......@@ -22,6 +22,7 @@
#include "dndTransport.h"
#include "dndVnodes.h"
#include "dndWorker.h"
#include "monitor.h"
static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg);
......@@ -473,19 +474,76 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
rpcSendResponse(&rpcRsp);
}
static int32_t dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
pInfo->dnode_id = dndGetDnodeId(pDnode);
tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
return 0;
}
static int32_t dndGetDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { return 0; }
static void dndSendMonitorReport(SDnode *pDnode) {
if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort);
SMonInfo *pMonitor = monCreateMonitorInfo();
if (pMonitor == NULL) return;
SMonBasicInfo basicInfo = {0};
if (dndGetBasicInfo(pDnode, &basicInfo) == 0) {
monSetBasicInfo(pMonitor, &basicInfo);
}
SMonClusterInfo clusterInfo = {0};
SMonVgroupInfo vgroupInfo = {0};
SMonGrantInfo grantInfo = {0};
if (dndGetMnodeMonitorInfo(pDnode, &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
monSetClusterInfo(pMonitor, &clusterInfo);
monSetVgroupInfo(pMonitor, &vgroupInfo);
monSetGrantInfo(pMonitor, &grantInfo);
}
SMonDnodeInfo dnodeInfo = {0};
if (dndGetDnodeInfo(pDnode, &dnodeInfo) == 0) {
monSetDnodeInfo(pMonitor, &dnodeInfo);
}
SMonDiskInfo diskInfo = {0};
if (dndGetDiskInfo(pDnode, &diskInfo) == 0) {
monSetDiskInfo(pMonitor, &diskInfo);
}
monSendReport(pMonitor);
monCleanupMonitorInfo(pMonitor);
}
static void *dnodeThreadRoutine(void *param) {
SDnode *pDnode = param;
SDnodeMgmt *pMgmt = &pDnode->dmgmt;
int32_t ms = tsStatusInterval * 1000;
int64_t lastStatusTime = taosGetTimestampMs();
int64_t lastMonitorTime = lastStatusTime;
setThreadName("dnode-hb");
while (true) {
pthread_testcancel();
taosMsleep(ms);
taosMsleep(200);
if (dndGetStat(pDnode) != DND_STAT_RUNNING || pMgmt->dropped) {
continue;
}
int64_t curTime = taosGetTimestampMs();
if (dndGetStat(pDnode) == DND_STAT_RUNNING && !pMgmt->statusSent && !pMgmt->dropped) {
float statusInterval = (curTime - lastStatusTime) / 1000.0f;
if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) {
dndSendStatusReq(pDnode);
lastStatusTime = curTime;
}
float monitorInterval = (curTime - lastMonitorTime) / 1000.0f;
if (monitorInterval >= tsMonitorInterval) {
dndSendMonitorReport(pDnode);
lastMonitorTime = curTime;
}
}
}
......
source/dnode/mgmt/impl/src/dndMnode.c
浏览文件 @ f0e392ae
......@@ -630,3 +630,13 @@ int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *enc
dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt);
return code;
}
int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL) return -1;
int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
dndReleaseMnode(pDnode, pMnode);
return code;
}
\ No newline at end of file
source/dnode/mgmt/impl/src/dndTransport.c
浏览文件 @ f0e392ae
......@@ -114,7 +114,6 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg;
/*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg;
......
source/dnode/mgmt/impl/src/dndVnodes.c
浏览文件 @ f0e392ae
......@@ -523,6 +523,9 @@ static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->walCfg.rollPeriod = 128;
pCfg->walCfg.segSize = 128;
pCfg->walCfg.vgId = pCreate->vgId;
pCfg->hashBegin = pCreate->hashBegin;
pCfg->hashEnd = pCreate->hashEnd;
pCfg->hashMethod = pCreate->hashMethod;
}
static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) {
......
source/dnode/mgmt/impl/test/sut/CMakeLists.txt
浏览文件 @ f0e392ae
aux_source_directory(src SUT_SRC)
add_library(sut STATIC ${SUT_SRC})
add_library(sut STATIC STATIC ${SUT_SRC})
target_link_libraries(
sut
PUBLIC dnode
......
source/dnode/mnode/impl/CMakeLists.txt
浏览文件 @ f0e392ae
aux_source_directory(src MNODE_SRC)
add_library(mnode ${MNODE_SRC})
add_library(mnode STATIC ${MNODE_SRC})
target_include_directories(
mnode
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mnode"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
mnode
PRIVATE scheduler
PRIVATE sdb
PRIVATE wal
PRIVATE transport
PRIVATE cjson
PRIVATE sync
mnode scheduler sdb wal transport cjson sync monitor
)
if(${BUILD_TEST})
......
source/dnode/mnode/impl/inc/mndInt.h
浏览文件 @ f0e392ae
......@@ -21,11 +21,11 @@
#include "sdb.h"
#include "tcache.h"
#include "tep.h"
#include "tglobal.h"
#include "tqueue.h"
#include "ttime.h"
#include "wal.h"
#include "version.h"
#include "tglobal.h"
#include "wal.h"
#ifdef __cplusplus
extern "C" {
......@@ -38,6 +38,20 @@ typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *p
typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter);
typedef struct SMnodeLoad {
int64_t numOfDnode;
int64_t numOfMnode;
int64_t numOfVgroup;
int64_t numOfDatabase;
int64_t numOfSuperTable;
int64_t numOfChildTable;
int64_t numOfNormalTable;
int64_t numOfColumn;
int64_t totalPoints;
int64_t totalStorage;
int64_t compStorage;
} SMnodeLoad;
typedef struct {
const char *name;
MndInitFp initFp;
......@@ -104,7 +118,9 @@ int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg);
void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg);
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
uint64_t mndGenerateUid(char *name, int32_t len) ;
uint64_t mndGenerateUid(char *name, int32_t len);
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
#ifdef __cplusplus
}
......
source/dnode/mnode/impl/src/mndDb.c
浏览文件 @ f0e392ae
......@@ -940,43 +940,51 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) {
char *p = strchr(usedbReq.db, '.');
if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
code = 0;
} else {
pDb = mndAcquireDb(pMnode, usedbReq.db);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto USE_DB_OVER;
}
pUser = mndAcquireUser(pMnode, pReq->user);
if (pUser == NULL) {
goto USE_DB_OVER;
}
memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
usedbRsp.uid = usedbReq.dbId;
usedbRsp.vgVersion = usedbReq.vgVersion;
if (mndCheckUseDbAuth(pUser, pDb) != 0) {
goto USE_DB_OVER;
}
mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr());
} else {
pUser = mndAcquireUser(pMnode, pReq->user);
if (pUser == NULL) {
goto USE_DB_OVER;
}
usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
if (usedbRsp.pVgroupInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto USE_DB_OVER;
}
if (mndCheckUseDbAuth(pUser, pDb) != 0) {
goto USE_DB_OVER;
}
if (usedbReq.vgVersion < pDb->vgVersion) {
mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos);
}
usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
if (usedbRsp.pVgroupInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto USE_DB_OVER;
}
if (usedbReq.vgVersion < pDb->vgVersion || usedbReq.dbId != pDb->uid) {
mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos);
}
memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN);
usedbRsp.uid = pDb->uid;
usedbRsp.vgVersion = pDb->vgVersion;
usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);
usedbRsp.hashMethod = pDb->hashMethod;
memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN);
usedbRsp.uid = pDb->uid;
usedbRsp.vgVersion = pDb->vgVersion;
usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);
usedbRsp.hashMethod = pDb->hashMethod;
code = 0;
}
}
int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp);
void *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto USE_DB_OVER;
}
......@@ -984,7 +992,6 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) {
pReq->pCont = pRsp;
pReq->contLen = contLen;
code = 0;
USE_DB_OVER:
if (code != 0) {
......
source/dnode/mnode/impl/src/mndSubscribe.c
浏览文件 @ f0e392ae
......@@ -270,9 +270,8 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
strcpy(rsp.cgroup, pReq->cgroup);
rsp.consumerId = consumerId;
rsp.epoch = pConsumer->epoch;
if (epoch != rsp.epoch) {
mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, rsp.epoch);
if (epoch != pConsumer->epoch) {
mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch);
SArray *pTopics = pConsumer->currentTopics;
int32_t sz = taosArrayGetSize(pTopics);
rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
......@@ -308,13 +307,16 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
mndReleaseSubscribe(pMnode, pSub);
}
}
int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
void *buf = rpcMallocCont(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
void *abuf = buf;
((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
((SMqRspHead *)buf)->epoch = pConsumer->epoch;
void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqCMGetSubEpRsp(&abuf, &rsp);
tDeleteSMqCMGetSubEpRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer);
......
source/dnode/mnode/impl/src/mndTelem.c
浏览文件 @ f0e392ae
......@@ -18,227 +18,68 @@
#include "mndCluster.h"
#include "mndSync.h"
#include "tbuffer.h"
#include "tversion.h"
#include "tjson.h"
#include "thttp.h"
#define TELEMETRY_SERVER "telemetry.taosdata.com"
#define TELEMETRY_PORT 80
#define REPORT_INTERVAL 86400
static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); }
static void mndCloseObject(SBufferWriter* bw) {
size_t len = tbufTell(bw);
if (tbufGetData(bw, false)[len - 1] == ',') {
tbufWriteCharAt(bw, len - 1, '}');
} else {
tbufWriteChar(bw, '}');
}
}
static void mndWriteString(SBufferWriter* bw, const char* str) {
tbufWriteChar(bw, '"');
tbufWrite(bw, str, strlen(str));
tbufWriteChar(bw, '"');
}
static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) {
mndWriteString(bw, k);
tbufWriteChar(bw, ':');
char buf[32] = {0};
sprintf(buf, "%" PRId64, v);
tbufWrite(bw, buf, strlen(buf));
tbufWriteChar(bw, ',');
}
static void mndAddStringField(SBufferWriter* bw, const char* k, const char* v) {
mndWriteString(bw, k);
tbufWriteChar(bw, ':');
mndWriteString(bw, v);
tbufWriteChar(bw, ',');
}
static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
int32_t done = 0;
// FILE* fp = fopen("/proc/cpuinfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
return;
}
while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
const char* v = strchr(line, ':') + 2;
mndAddStringField(bw, "cpuModel", v);
done |= 1;
} else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) {
const char* v = strchr(line, ':') + 2;
mndWriteString(bw, "numOfCpu");
tbufWriteChar(bw, ':');
tbufWrite(bw, v, strlen(v));
tbufWriteChar(bw, ',');
done |= 2;
}
}
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
// FILE* fp = fopen("/etc/os-release", "r");
TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
return;
}
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "PRETTY_NAME", 11) == 0) {
const char* p = strchr(line, '=') + 1;
if (*p == '"') {
p++;
line[size - 2] = 0;
}
mndAddStringField(bw, "os", p);
break;
}
}
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
// FILE* fp = fopen("/proc/meminfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
return;
}
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "MemTotal", 8) == 0) {
const char* p = strchr(line, ':') + 1;
while (*p == ' ') p++;
mndAddStringField(bw, "memory", p);
break;
}
}
if(line != NULL) free(line);
taosCloseFile(&pFile);
static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) {
SMnodeLoad load = {0};
if (mndGetLoad(pMnode, &load) != 0) return;
tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode);
tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode);
tjsonAddDoubleToObject(pJson, "numOfVgroup", load.numOfVgroup);
tjsonAddDoubleToObject(pJson, "numOfDatabase", load.numOfDatabase);
tjsonAddDoubleToObject(pJson, "numOfSuperTable", load.numOfSuperTable);
tjsonAddDoubleToObject(pJson, "numOfChildTable", load.numOfChildTable);
tjsonAddDoubleToObject(pJson, "numOfColumn", load.numOfColumn);
tjsonAddDoubleToObject(pJson, "numOfPoint", load.totalPoints);
tjsonAddDoubleToObject(pJson, "totalStorage", load.totalStorage);
tjsonAddDoubleToObject(pJson, "compStorage", load.compStorage);
}
static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) {
static char* mndBuildTelemetryReport(SMnode* pMnode) {
char tmp[4096] = {0};
STelemMgmt* pMgmt = &pMnode->telemMgmt;
mndAddStringField(bw, "version", version);
mndAddStringField(bw, "buildInfo", buildinfo);
mndAddStringField(bw, "gitInfo", gitinfo);
mndAddStringField(bw, "email", pMgmt->email);
}
static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) {
SMnodeLoad load = {0};
if (mndGetLoad(pMnode, &load) != 0) {
return;
}
mndAddIntField(bw, "numOfDnode", load.numOfDnode);
mndAddIntField(bw, "numOfMnode", load.numOfMnode);
mndAddIntField(bw, "numOfVgroup", load.numOfVgroup);
mndAddIntField(bw, "numOfDatabase", load.numOfDatabase);
mndAddIntField(bw, "numOfSuperTable", load.numOfSuperTable);
mndAddIntField(bw, "numOfChildTable", load.numOfChildTable);
mndAddIntField(bw, "numOfColumn", load.numOfColumn);
mndAddIntField(bw, "numOfPoint", load.totalPoints);
mndAddIntField(bw, "totalStorage", load.totalStorage);
mndAddIntField(bw, "compStorage", load.compStorage);
}
static void mndSendTelemetryReport(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
SBufferWriter bw = tbufInitWriter(NULL, false);
int32_t code = -1;
char buf[128] = {0};
SOCKET fd = 0;
SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return NULL;
char clusterName[64] = {0};
if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) {
goto SEND_OVER;
}
mndBeginObject(&bw);
mndAddStringField(&bw, "instanceId", clusterName);
mndAddIntField(&bw, "reportVersion", 1);
mndAddOsInfo(pMnode, &bw);
mndAddCpuInfo(pMnode, &bw);
mndAddMemoryInfo(pMnode, &bw);
mndAddVersionInfo(pMnode, &bw);
mndAddRuntimeInfo(pMnode, &bw);
mndCloseObject(&bw);
mndGetClusterName(pMnode, clusterName, sizeof(clusterName));
tjsonAddStringToObject(pJson, "instanceId", clusterName);
tjsonAddDoubleToObject(pJson, "reportVersion", 1);
uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
if (ip == 0xffffffff) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr());
goto SEND_OVER;
if (taosGetOsReleaseName(tmp, sizeof(tmp))) {
tjsonAddStringToObject(pJson, "os", tmp);
}
fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0);
if (fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr());
goto SEND_OVER;
}
const char* header =
"POST /report HTTP/1.1\n"
"Host: " TELEMETRY_SERVER
"\n"
"Content-Type: application/json\n"
"Content-Length: ";
if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) {
goto SEND_OVER;
}
int32_t contLen = (int32_t)(tbufTell(&bw));
sprintf(buf, "%d\n\n", contLen);
if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) {
goto SEND_OVER;
}
const char* pCont = tbufGetData(&bw, false);
if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) {
goto SEND_OVER;
int32_t numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) {
tjsonAddStringToObject(pJson, "cpuModel", tmp);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
} else {
tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores());
}
// read something to avoid nginx error 499
if (taosReadSocket(fd, buf, 10) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to receive response since %s", terrstr());
goto SEND_OVER;
uint64_t memoryKB = 0;
if (taosGetTotalSysMemoryKB(&memoryKB)) {
snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
}
mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont);
code = 0;
tjsonAddStringToObject(pJson, "version", version);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo);
tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
tjsonAddStringToObject(pJson, "email", pMgmt->email);
SEND_OVER:
tbufCloseWriter(&bw);
taosCloseSocket(fd);
mndBuildRuntimeInfo(pMnode, pJson);
if (code != 0) {
mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr());
}
char* pCont = tjsonToString(pJson);
tjsonDelete(pJson);
return pCont;
}
static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
......@@ -247,33 +88,23 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
if (!pMgmt->enable) return 0;
taosWLockLatch(&pMgmt->lock);
mndSendTelemetryReport(pMnode);
char* pCont = mndBuildTelemetryReport(pMnode);
if (pCont != NULL) {
taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont));
free(pCont);
}
taosWUnLockLatch(&pMgmt->lock);
return 0;
}
static void mndGetEmail(SMnode* pMnode, char* filepath) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) {
return;
}
if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
}
taosCloseFile(&pFile);
}
int32_t mndInitTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
pMgmt->enable = tsEnableTelemetryReporting;
taosInitRWLatch(&pMgmt->lock);
mndGetEmail(pMnode, "/usr/local/taos/email");
taosInitRWLatch(&pMgmt->lock);
pMgmt->enable = tsEnableTelemetryReporting;
taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
mDebug("mnode telemetry is initialized");
return 0;
}
......
source/dnode/mnode/impl/src/mndTopic.c
浏览文件 @ f0e392ae
......@@ -72,7 +72,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
if (pRaw == NULL) goto TOPIC_ENCODE_OVER;
int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER);
......@@ -121,7 +121,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
int32_t len;
int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->createTime, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER);
......@@ -206,7 +206,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) {
SName name = {0};
tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
char db[TSDB_TABLE_FNAME_LEN] = {0};
char db[TSDB_TOPIC_FNAME_LEN] = {0};
tNameGetFullDbName(&name, db);
return mndAcquireDb(pMnode, db);
......@@ -223,7 +223,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
pDrop->head.contLen = htonl(contLen);
pDrop->head.vgId = htonl(pVgroup->vgId);
memcpy(pDrop->name, pTopic->name, TSDB_TABLE_FNAME_LEN);
memcpy(pDrop->name, pTopic->name, TSDB_TOPIC_FNAME_LEN);
pDrop->tuid = htobe64(pTopic->uid);
return pDrop;
......@@ -343,6 +343,7 @@ CREATE_TOPIC_OVER:
}
static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic) {
// TODO: cannot drop when subscribed
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_TOPIC, &pReq->rpcMsg);
if (pTrans == NULL) {
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
......@@ -408,75 +409,34 @@ static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) {
return 0;
}
static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq) {
SMnode *pMnode = pReq->pMnode;
STableInfoReq infoReq = {0};
if (tSerializeSTableInfoReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &infoReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mDebug("topic:%s, start to retrieve meta", infoReq.tbName);
#if 0
SDbObj *pDb = mndAcquireDbByTopic(pMnode, pInfo->tableFname);
static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
mError("topic:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr());
return -1;
}
STopicObj *pTopic = mndAcquireTopic(pMnode, pInfo->tableFname);
if (pTopic == NULL) {
mndReleaseDb(pMnode, pDb);
terrno = TSDB_CODE_MND_INVALID_TOPIC;
mError("topic:%s, failed to get meta since %s", pInfo->tableFname, terrstr());
return -1;
}
int32_t numOfTopics = 0;
void *pIter = NULL;
while (1) {
SMqTopicObj *pTopic = NULL;
pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
if (pIter == NULL) break;
taosRLockLatch(&pTopic->lock);
int32_t totalCols = pTopic->numOfColumns + pTopic->numOfTags;
int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema);
if (pTopic->dbUid == pDb->uid) {
numOfTopics++;
}
STableMetaRsp *pMeta = rpcMallocCont(contLen);
if (pMeta == NULL) {
taosRUnLockLatch(&pTopic->lock);
mndReleaseDb(pMnode, pDb);
mndReleaseTopic(pMnode, pTopic);
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("topic:%s, failed to get meta since %s", pInfo->tableFname, terrstr());
return -1;
sdbRelease(pSdb, pTopic);
}
memcpy(pMeta->topicFname, pTopic->name, TSDB_TABLE_FNAME_LEN);
pMeta->numOfTags = htonl(pTopic->numOfTags);
pMeta->numOfColumns = htonl(pTopic->numOfColumns);
pMeta->precision = pDb->cfg.precision;
pMeta->tableType = TSDB_SUPER_TABLE;
pMeta->update = pDb->cfg.update;
pMeta->sversion = htonl(pTopic->version);
pMeta->tuid = htonl(pTopic->uid);
for (int32_t i = 0; i < totalCols; ++i) {
SSchema *pSchema = &pMeta->pSchemas[i];
SSchema *pSrcSchema = &pTopic->pSchema[i];
memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
pSchema->type = pSrcSchema->type;
pSchema->colId = htonl(pSrcSchema->colId);
pSchema->bytes = htonl(pSrcSchema->bytes);
}
taosRUnLockLatch(&pTopic->lock);
*pNumOfTopics = numOfTopics;
mndReleaseDb(pMnode, pDb);
mndReleaseTopic(pMnode, pTopic);
pReq->pCont = pMeta;
pReq->contLen = contLen;
mDebug("topic:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pTopic->numOfColumns, pTopic->numOfTags);
#endif
return 0;
}
#endif
static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
SSdb *pSdb = pMnode->pSdb;
......@@ -571,7 +531,7 @@ static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, in
if (pTopic->dbUid != pDb->uid) {
if (strncmp(pTopic->name, prefix, prefixLen) != 0) {
mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid);
mError("Inconsistent topic data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid);
}
sdbRelease(pSdb, pTopic);
......@@ -580,8 +540,8 @@ static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, in
cols = 0;
char topicName[TSDB_TABLE_NAME_LEN] = {0};
tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TABLE_NAME_LEN);
char topicName[TSDB_TOPIC_NAME_LEN] = {0};
tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TOPIC_NAME_LEN);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, topicName);
cols++;
......
source/dnode/mnode/impl/src/mndVgroup.c
浏览文件 @ f0e392ae
......@@ -215,6 +215,9 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
createReq.replica = pVgroup->replica;
createReq.selfIndex = -1;
createReq.streamMode = pVgroup->streamMode;
createReq.hashBegin = pVgroup->hashBegin;
createReq.hashEnd = pVgroup->hashEnd;
createReq.hashMethod = pDb->hashMethod;
for (int32_t v = 0; v < pVgroup->replica; ++v) {
SReplica *pReplica = &createReq.replicas[v];
......
source/dnode/mnode/impl/src/mnode.c
浏览文件 @ f0e392ae
......@@ -22,6 +22,7 @@
#include "mndDb.h"
#include "mndDnode.h"
#include "mndFunc.h"
#include "mndInfoSchema.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndProfile.h"
......@@ -36,7 +37,6 @@
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "mndInfoSchema.h"
#define MQ_TIMER_MS 3000
#define TRNAS_TIMER_MS 6000
......@@ -400,6 +400,11 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
return 0;
}
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
return 0;
}
SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg));
if (pMsg == NULL) {
......@@ -489,7 +494,7 @@ PROCESS_RPC_END:
if (code == TSDB_CODE_APP_NOT_READY) {
mndSendRedirectRsp(pMnode, &pMsg->rpcMsg);
} else if (code != 0) {
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .code = code};
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont, .code = code};
rpcSendResponse(&rpcRsp);
} else {
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont};
......@@ -505,10 +510,9 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
}
}
// Note: uid 0 is reserved
uint64_t mndGenerateUid(char *name, int32_t len) {
int32_t hashval = MurmurHash3_32(name, len);
int32_t hashval = MurmurHash3_32(name, len);
do {
int64_t us = taosGetTimestampUs();
......
source/dnode/mnode/sdb/CMakeLists.txt
浏览文件 @ f0e392ae
aux_source_directory(src MNODE_SRC)
add_library(sdb ${MNODE_SRC})
add_library(sdb STATIC ${MNODE_SRC})
target_include_directories(
sdb
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mnode/sdb"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
sdb
PRIVATE os
PRIVATE common
PRIVATE util
sdb os common util
)
\ No newline at end of file
source/dnode/qnode/CMakeLists.txt
浏览文件 @ f0e392ae
aux_source_directory(src QNODE_SRC)
add_library(qnode ${QNODE_SRC})
add_library(qnode STATIC ${QNODE_SRC})
target_include_directories(
qnode
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/qnode"
......
source/dnode/snode/CMakeLists.txt
浏览文件 @ f0e392ae
aux_source_directory(src SNODE_SRC)
add_library(snode ${SNODE_SRC})
add_library(snode STATIC ${SNODE_SRC})
target_include_directories(
snode
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/snode"
......
source/dnode/vnode/inc/tq.h
浏览文件 @ f0e392ae
......@@ -51,7 +51,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
void tqClose(STQ*);
// required by vnode
int tqPushMsg(STQ*, void* msg, int64_t version);
int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version);
int tqCommit(STQ*);
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg);
......
source/dnode/vnode/inc/tsdb.h
浏览文件 @ f0e392ae
......@@ -27,12 +27,12 @@ extern "C" {
typedef struct SDataStatis {
int16_t colId;
int64_t sum;
int64_t max;
int64_t min;
int16_t maxIndex;
int16_t minIndex;
int16_t numOfNull;
int64_t sum;
int64_t max;
int64_t min;
} SDataStatis;
typedef struct STable {
......@@ -53,6 +53,8 @@ typedef struct STsdb STsdb;
typedef struct STsdbCfg {
int8_t precision;
int8_t update;
int8_t compression;
uint64_t lruCacheSize;
int32_t daysPerFile;
int32_t minRowsPerFileBlock;
......@@ -60,8 +62,6 @@ typedef struct STsdbCfg {
int32_t keep;
int32_t keep1;
int32_t keep2;
int8_t update;
int8_t compression;
} STsdbCfg;
// query condition to build multi-table data block iterator
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ f0e392ae
......@@ -57,13 +57,12 @@ typedef struct {
SMetaCfg metaCfg;
STqCfg tqCfg;
SWalCfg walCfg;
uint32_t hashBegin;
uint32_t hashEnd;
int8_t hashMethod;
} SVnodeCfg;
typedef struct {
int32_t sver;
const char *timezone;
const char *locale;
const char *charset;
uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO)
PutReqToVQueryQFp putReqToVQueryQFp;
SendReqToDnodeFp sendReqToDnodeFp;
......
source/dnode/vnode/src/inc/tqInt.h
浏览文件 @ f0e392ae
......@@ -79,19 +79,19 @@ extern int32_t tqDebugFlag;
// 4096 - 4080
#define TQ_IDX_PAGE_HEAD_SIZE 16
#define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1
#define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1
#define TQ_ACTION_INUSE_CONT 2
#define TQ_ACTION_INTXN 3
#define TQ_ACTION_INTXN 3
#define TQ_SVER 0
// TODO: inplace mode is not implemented
#define TQ_UPDATE_INPLACE 0
#define TQ_UPDATE_APPEND 1
#define TQ_UPDATE_APPEND 1
#define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2
#define TQ_DUP_INTXN_REJECT 2
static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
......@@ -160,7 +160,7 @@ struct STQ {
STqMemRef tqMemRef;
STqMetaStore* tqMeta;
SWal* pWal;
SMeta* pMeta;
SMeta* pVnodeMeta;
};
typedef struct {
......@@ -190,9 +190,6 @@ typedef struct {
char* logicalPlan;
char* physicalPlan;
char* qmsg;
int64_t persistedOffset;
int64_t committedOffset;
int64_t currentOffset;
STqBuffer buffer;
SWalReadHandle* pReadhandle;
} STqTopic;
......@@ -201,7 +198,7 @@ typedef struct {
int64_t consumerId;
int64_t epoch;
char cgroup[TSDB_TOPIC_FNAME_LEN];
SArray* topics; // SArray<STqTopicHandle>
SArray* topics; // SArray<STqTopic>
} STqConsumer;
int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**);
......
source/dnode/vnode/src/inc/tsdbFS.h
浏览文件 @ f0e392ae
......@@ -18,8 +18,6 @@
#include "tsdbFile.h"
#define TSDB_FS_VERSION 0
// ================== TSDB global config
extern bool tsdbForceKeepFile;
......
source/dnode/vnode/src/inc/tsdbFile.h
浏览文件 @ f0e392ae
......@@ -44,7 +44,37 @@
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T;
typedef enum {
TSDB_FILE_HEAD = 0, // .head
TSDB_FILE_DATA, // .data
TSDB_FILE_LAST, // .last
TSDB_FILE_SMAD, // .smad(Block-wise SMA)
TSDB_FILE_SMAL, // .smal(Block-wise SMA)
TSDB_FILE_MAX, //
TSDB_FILE_META, // meta
TSDB_FILE_TSMA, // .tsma.${sma_index_name}, Time-range-wise SMA
TSDB_FILE_RSMA, // .rsma.${sma_index_name}, Time-range-wise Rollup SMA
} TSDB_FILE_T;
typedef enum {
TSDB_FS_VER_0 = 0,
TSDB_FS_VER_MAX,
} ETsdbFsVer;
#define TSDB_LATEST_FVER TSDB_FS_VER_0 // latest version for DFile
#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0 // latest version for 'current' file
static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest version for DFile
switch (fType) {
case TSDB_FILE_HEAD: // .head
case TSDB_FILE_DATA: // .data
case TSDB_FILE_LAST: // .last
case TSDB_FILE_SMAD: // .smad(Block-wise SMA)
case TSDB_FILE_SMAL: // .smal(Block-wise SMA)
default:
return TSDB_LATEST_FVER;
}
}
#if 0
// =============== SMFile
......@@ -169,6 +199,7 @@ static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nby
// =============== SDFile
typedef struct {
uint32_t magic;
uint32_t fver;
uint32_t len;
uint32_t totalBlocks;
uint32_t totalSubBlocks;
......@@ -188,7 +219,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t
void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
int tsdbEncodeSDFile(void** buf, SDFile* pDFile);
void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile);
int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader);
int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType);
int tsdbUpdateDFileHeader(SDFile* pDFile);
int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
......@@ -292,12 +323,18 @@ static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
// =============== SDFileSet
typedef struct {
int fid;
int state;
SDFile files[TSDB_FILE_MAX];
int fid;
int8_t state; // -128~127
uint8_t ver; // 0~255, DFileSet version
uint16_t reserve;
SDFile files[TSDB_FILE_MAX];
} SDFileSet;
#define TSDB_LATEST_FSET_VER 0
#define TSDB_FSET_FID(s) ((s)->fid)
#define TSDB_FSET_STATE(s) ((s)->state)
#define TSDB_FSET_VER(s) ((s)->ver)
#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
......
source/dnode/vnode/src/inc/tsdbReadImpl.h
浏览文件 @ f0e392ae
......@@ -36,6 +36,7 @@ typedef struct {
TSKEY maxKey;
} SBlockIdx;
#ifdef TD_REFACTOR_3
typedef struct {
int64_t last : 1;
int64_t offset : 63;
......@@ -49,22 +50,35 @@ typedef struct {
TSKEY keyLast;
} SBlock;
#else
typedef enum {
TSDB_SBLK_VER_0 = 0,
TSDB_SBLK_VER_MAX,
} ESBlockVer;
#define SBlockVerLatest TSDB_SBLK_VER_0
typedef struct {
int64_t last : 1;
int64_t offset : 63;
int32_t algorithm : 8;
int32_t numOfRows : 24;
uint8_t reserve0;
uint8_t last : 1;
uint8_t blkVer : 7;
uint8_t numOfSubBlocks;
int16_t numOfCols; // not including timestamp column
uint32_t len : 32; // data block length
uint32_t len; // data block length
uint32_t keyLen : 24; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
uint32_t reserve1 : 8;
uint64_t blkVer : 8;
uint64_t aggrOffset : 56;
uint32_t reserve : 8;
int32_t algorithm : 8;
int32_t numOfRows : 24;
int64_t offset;
uint64_t aggrStat : 1;
uint64_t aggrOffset : 63;
TSKEY keyFirst;
TSKEY keyLast;
} SBlock_3;
} SBlockV0;
#define SBlock SBlockV0 // latest SBlock definition
#endif
typedef struct {
int32_t delimiter; // For recovery usage
......@@ -73,6 +87,7 @@ typedef struct {
SBlock blocks[];
} SBlockInfo;
#ifdef TD_REFACTOR_3
typedef struct {
int16_t colId;
uint16_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
......@@ -89,17 +104,50 @@ typedef struct {
uint8_t offsetH;
char padding[1];
} SBlockCol;
#else
typedef struct {
int16_t colId;
uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
uint8_t reserve : 7;
uint8_t type;
int32_t len;
uint32_t offset;
} SBlockColV0;
#define SBlockCol SBlockColV0 // latest SBlockCol definition
typedef struct {
int16_t colId;
int16_t maxIndex;
int16_t minIndex;
int16_t numOfNull;
int64_t sum;
int64_t max;
int64_t min;
} SAggrBlkColV0;
#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition
#endif
// Code here just for back-ward compatibility
static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
#ifdef TD_REFACTOR_3
pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
pBlockCol->offsetH = (uint8_t)(offset >> 24);
#else
pBlockCol->offset = offset;
#endif
}
static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
#ifdef TD_REFACTOR_3
uint32_t offset1 = pBlockCol->offset;
uint32_t offset2 = pBlockCol->offsetH;
return (offset1 | (offset2 << 24));
#else
return pBlockCol->offset;
#endif
}
typedef struct {
......@@ -109,31 +157,57 @@ typedef struct {
SBlockCol cols[];
} SBlockData;
typedef void SAggrBlkData; // SBlockCol cols[];
struct SReadH {
STsdb * pRepo;
SDFileSet rSet; // FSET to read
SArray * aBlkIdx; // SBlockIdx array
STable * pTable; // table to read
SBlockIdx * pBlkIdx; // current reading table SBlockIdx
int cidx;
SBlockInfo *pBlkInfo;
SBlockData *pBlkData; // Block info
SDataCols * pDCols[2];
void * pBuf; // buffer
void * pCBuf; // compression buffer
STsdb * pRepo;
SDFileSet rSet; // FSET to read
SArray * aBlkIdx; // SBlockIdx array
STable * pTable; // table to read
SBlockIdx * pBlkIdx; // current reading table SBlockIdx
int cidx;
SBlockInfo * pBlkInfo;
SBlockData * pBlkData; // Block info
SAggrBlkData *pAggrBlkData; // Aggregate Block info
SDataCols * pDCols[2];
void * pBuf; // buffer
void * pCBuf; // compression buffer
void * pExBuf; // extra buffer
};
#define TSDB_READ_REPO(rh) ((rh)->pRepo)
#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
#define TSDB_READ_FSET(rh) (&((rh)->rSet))
#define TSDB_READ_TABLE(rh) ((rh)->pTable)
#define TSDB_READ_REPO(rh) ((rh)->pRepo)
#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
#define TSDB_READ_FSET(rh) (&((rh)->rSet))
#define TSDB_READ_TABLE(rh) ((rh)->pTable)
#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
#define TSDB_READ_BUF(rh) ((rh)->pBuf)
#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf)
#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD)
#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL)
#define TSDB_READ_BUF(rh) ((rh)->pBuf)
#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf)
#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf)
#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \
(sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM))
static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
switch (blkVer) {
case TSDB_SBLK_VER_0:
default:
return TSDB_BLOCK_STATIS_SIZE(nCols, 0);
}
}
#define TSDB_BLOCK_STATIS_SIZE(ncols) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM))
#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM))
static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
switch (blkVer) {
case TSDB_SBLK_VER_0:
default:
return TSDB_BLOCK_AGGR_SIZE(nCols, 0);
}
}
int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo);
void tsdbDestroyReadH(SReadH *pReadh);
......@@ -147,7 +221,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo
int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols);
void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
void * pBuf = *ppBuf;
......
source/dnode/vnode/src/tq/tq.c
浏览文件 @ f0e392ae
......@@ -42,7 +42,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
pTq->path = strdup(path);
pTq->tqConfig = tqConfig;
pTq->pWal = pWal;
pTq->pMeta = pMeta;
pTq->pVnodeMeta = pMeta;
#if 0
pTq->tqMemRef.pAllocatorFactory = allocFac;
pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
......@@ -71,9 +71,11 @@ void tqClose(STQ* pTq) {
// TODO
}
int tqPushMsg(STQ* pTq, void* p, int64_t version) {
// add reference
// judge and launch new query
int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
// TODO: add reference
// if handle waiting, launch query and response to consumer
//
// if no waiting handle, return
return 0;
}
......@@ -101,9 +103,9 @@ static FORCE_INLINE int32_t tEncodeSTqTopic(void** buf, const STqTopic* pTopic)
/*tlen += taosEncodeString(buf, pTopic->logicalPlan);*/
/*tlen += taosEncodeString(buf, pTopic->physicalPlan);*/
tlen += taosEncodeString(buf, pTopic->qmsg);
tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset);
tlen += taosEncodeFixedI64(buf, pTopic->committedOffset);
tlen += taosEncodeFixedI64(buf, pTopic->currentOffset);
/*tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset);*/
/*tlen += taosEncodeFixedI64(buf, pTopic->committedOffset);*/
/*tlen += taosEncodeFixedI64(buf, pTopic->currentOffset);*/
return tlen;
}
......@@ -113,9 +115,9 @@ static FORCE_INLINE const void* tDecodeSTqTopic(const void* buf, STqTopic* pTopi
/*buf = taosDecodeString(buf, &pTopic->logicalPlan);*/
/*buf = taosDecodeString(buf, &pTopic->physicalPlan);*/
buf = taosDecodeString(buf, &pTopic->qmsg);
buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset);
buf = taosDecodeFixedI64(buf, &pTopic->committedOffset);
buf = taosDecodeFixedI64(buf, &pTopic->currentOffset);
/*buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset);*/
/*buf = taosDecodeFixedI64(buf, &pTopic->committedOffset);*/
/*buf = taosDecodeFixedI64(buf, &pTopic->currentOffset);*/
return buf;
}
......@@ -194,8 +196,8 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
}
for (int j = 0; j < TQ_BUFFER_SIZE; j++) {
pTopic->buffer.output[j].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta);
SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta};
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta};
pTopic->buffer.output[j].pReadHandle = pReadHandle;
pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
}
......@@ -218,7 +220,11 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
fetchOffset = pReq->currentOffset + 1;
}
SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL};
SMqConsumeRsp rsp = {
.consumerId = consumerId,
.numOfTopics = 0,
.pBlockData = NULL,
};
STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
if (pConsumer == NULL) {
......@@ -243,7 +249,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
// TODO: no more log, set timer to wait blocking time
// if data inserted during waiting, launch query and
// rsponse to user
// response to user
break;
}
pHead = pTopic->pReadhandle->pHead;
......@@ -268,19 +274,20 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
taosArrayPush(pRes, pDataBlock);
rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper;
rsp.rspOffset = fetchOffset;
pTopic->currentOffset = fetchOffset;
rsp.numOfTopics = 1;
rsp.pBlockData = pRes;
int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp);
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
return -1;
}
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
((SMqRspHead*)buf)->epoch = pReq->epoch;
void* abuf = buf;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqConsumeRsp(&abuf, &rsp);
taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);
pMsg->pCont = buf;
......@@ -295,14 +302,16 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
}
}
int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp);
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
return -1;
}
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
((SMqRspHead*)buf)->epoch = pReq->epoch;
void* abuf = buf;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqConsumeRsp(&abuf, &rsp);
rsp.pBlockData = NULL;
pMsg->pCont = buf;
......@@ -312,158 +321,6 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
return 0;
}
#if 0
int32_t tqProcessConsumeReqV0(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont;
int64_t reqId = pReq->reqId;
int64_t consumerId = pReq->consumerId;
int64_t fetchOffset = pReq->offset;
int64_t blockingTime = pReq->blockingTime;
SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL};
/*printf("vg %d get consume req\n", pReq->head.vgId);*/
STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
if (pConsumer == NULL) {
pMsg->pCont = NULL;
pMsg->contLen = 0;
pMsg->code = -1;
rpcSendResponse(pMsg);
return 0;
}
int sz = taosArrayGetSize(pConsumer->topics);
for (int i = 0; i < sz; i++) {
STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
// TODO: support multiple topic in one req
if (strcmp(pTopic->topicName, pReq->topic) != 0) {
ASSERT(false);
continue;
}
if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) {
pTopic->committedOffset = pReq->offset;
pMsg->pCont = NULL;
pMsg->contLen = 0;
pMsg->code = 0;
rpcSendResponse(pMsg);
return 0;
}
if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) {
pTopic->committedOffset = pReq->offset - 1;
}
rsp.committedOffset = pTopic->committedOffset;
rsp.reqOffset = pReq->offset;
rsp.skipLogNum = 0;
if (fetchOffset <= pTopic->committedOffset) {
fetchOffset = pTopic->committedOffset + 1;
}
/*printf("vg %d fetch Offset %ld\n", pReq->head.vgId, fetchOffset);*/
int8_t pos;
int8_t skip = 0;
SWalHead* pHead;
while (1) {
pos = fetchOffset % TQ_BUFFER_SIZE;
skip = atomic_val_compare_exchange_8(&pTopic->buffer.output[pos].status, 0, 1);
if (skip == 1) {
// do nothing
break;
}
if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
printf("read offset %ld\n", fetchOffset);
// check err
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
skip = 1;
break;
}
// read until find TDMT_VND_SUBMIT
pHead = pTopic->pReadhandle->pHead;
if (pHead->head.msgType == TDMT_VND_SUBMIT) {
}
rsp.skipLogNum++;
if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
printf("read offset %ld\n", fetchOffset);
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
skip = 1;
break;
}
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
fetchOffset++;
}
if (skip == 1) continue;
SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body;
qTaskInfo_t task = pTopic->buffer.output[pos].task;
printf("current fetch offset %ld\n", fetchOffset);
qSetStreamInput(task, pCont);
// SArray<SSDataBlock>
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
while (1) {
SSDataBlock* pDataBlock;
uint64_t ts;
if (qExecTask(task, &pDataBlock, &ts) < 0) {
break;
}
if (pDataBlock != NULL) {
taosArrayPush(pRes, pDataBlock);
} else {
break;
}
}
// TODO copy
rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper;
rsp.rspOffset = fetchOffset;
pTopic->currentOffset = fetchOffset;
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
if (taosArrayGetSize(pRes) == 0) {
taosArrayDestroy(pRes);
fetchOffset++;
continue;
} else {
rsp.numOfTopics++;
}
rsp.pBlockData = pRes;
#if 0
pTopic->buffer.output[pos].dst = pRes;
if (pTopic->buffer.firstOffset == -1 || pReq->offset < pTopic->buffer.firstOffset) {
pTopic->buffer.firstOffset = pReq->offset;
}
if (pTopic->buffer.lastOffset == -1 || pReq->offset > pTopic->buffer.lastOffset) {
pTopic->buffer.lastOffset = pReq->offset;
}
#endif
}
int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
return -1;
}
void* abuf = buf;
tEncodeSMqConsumeRsp(&abuf, &rsp);
if (rsp.pBlockData) {
taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);
rsp.pBlockData = NULL;
}
pMsg->pCont = buf;
pMsg->contLen = tlen;
pMsg->code = 0;
rpcSendResponse(pMsg);
return 0;
}
#endif
int32_t tqProcessRebReq(STQ* pTq, char* msg) {
SMqMVRebReq req = {0};
tDecodeSMqMVRebReq(msg, &req);
......@@ -505,8 +362,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
pTopic->logicalPlan = req.logicalPlan;
pTopic->physicalPlan = req.physicalPlan;
pTopic->qmsg = req.qmsg;
pTopic->committedOffset = -1;
pTopic->currentOffset = -1;
/*pTopic->committedOffset = -1;*/
/*pTopic->currentOffset = -1;*/
pTopic->buffer.firstOffset = -1;
pTopic->buffer.lastOffset = -1;
......@@ -516,8 +373,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
}
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
pTopic->buffer.output[i].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta);
SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta};
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta};
pTopic->buffer.output[i].pReadHandle = pReadHandle;
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
}
......
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ f0e392ae
......@@ -50,8 +50,11 @@ typedef struct {
#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD)
#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA)
#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST)
#define TSDB_COMMIT_SMAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD)
#define TSDB_COMMIT_SMAL_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL)
#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh))
#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh))
#define TSDB_COMMIT_EXBUF(ch) TSDB_READ_EXBUF(&((ch)->readh))
#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock)
#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
......@@ -509,7 +512,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
// TSDB_FILE_HEAD
SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith);
tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD);
if (tsdbCreateDFile(pRepo, pWHeadf, true) < 0) {
if (tsdbCreateDFile(pRepo, pWHeadf, true, TSDB_FILE_HEAD) < 0) {
tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf),
tstrerror(terrno));
......@@ -560,7 +563,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
tsdbInitDFile(pRepo, pWLastf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST);
pCommith->isLFileSame = false;
if (tsdbCreateDFile(pRepo, pWLastf, true) < 0) {
if (tsdbCreateDFile(pRepo, pWLastf, true, TSDB_FILE_LAST) < 0) {
tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf),
tstrerror(terrno));
......@@ -572,6 +575,74 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
}
}
}
// TSDB_FILE_SMAD
SDFile *pRSmadF = TSDB_READ_SMAD_FILE(&(pCommith->readh));
SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith);
if (access(TSDB_FILE_FULL_NAME(pRSmadF), F_OK) != 0) {
tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF));
tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD);
if (tsdbCreateDFile(pRepo, pWSmadF, true, TSDB_FILE_SMAD) < 0) {
tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF),
tstrerror(terrno));
tsdbCloseDFileSet(pWSet);
(void)tsdbRemoveDFile(pWHeadf);
if (pCommith->isRFileSet) {
tsdbCloseAndUnsetFSet(&(pCommith->readh));
return -1;
}
}
} else {
tsdbInitDFileEx(pWSmadF, pRSmadF);
if (tsdbOpenDFile(pWSmadF, O_RDWR) < 0) {
tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF),
tstrerror(terrno));
tsdbCloseDFileSet(pWSet);
tsdbRemoveDFile(pWHeadf);
if (pCommith->isRFileSet) {
tsdbCloseAndUnsetFSet(&(pCommith->readh));
return -1;
}
}
}
// TSDB_FILE_SMAL
SDFile *pRSmalF = TSDB_READ_SMAL_FILE(&(pCommith->readh));
SDFile *pWSmalF = TSDB_COMMIT_SMAL_FILE(pCommith);
if ((pCommith->isLFileSame) && access(TSDB_FILE_FULL_NAME(pRSmalF), F_OK) == 0) {
tsdbInitDFileEx(pWSmalF, pRSmalF);
if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) {
tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF),
tstrerror(terrno));
tsdbCloseDFileSet(pWSet);
tsdbRemoveDFile(pWHeadf);
if (pCommith->isRFileSet) {
tsdbCloseAndUnsetFSet(&(pCommith->readh));
return -1;
}
}
} else {
tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmalF));
tsdbInitDFile(pRepo, pWSmalF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAL);
if (tsdbCreateDFile(pRepo, pWSmalF, true, TSDB_FILE_SMAL) < 0) {
tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF),
tstrerror(terrno));
tsdbCloseDFileSet(pWSet);
(void)tsdbRemoveDFile(pWHeadf);
if (pCommith->isRFileSet) {
tsdbCloseAndUnsetFSet(&(pCommith->readh));
return -1;
}
}
}
}
return 0;
......@@ -1131,41 +1202,57 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) {
}
}
int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast,
bool isSuper, void **ppBuf, void **ppCBuf) {
STsdbCfg *pCfg = REPO_CFG(pRepo);
SBlockData *pBlockData;
int64_t offset = 0;
int rowsToWrite = pDataCols->numOfRows;
int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) {
STsdbCfg * pCfg = REPO_CFG(pRepo);
SBlockData * pBlockData = NULL;
SAggrBlkData *pAggrBlkData = NULL;
int64_t offset = 0, offsetAggr = 0;
int rowsToWrite = pDataCols->numOfRows;
ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock);
ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock);
// Make buffer space
if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) {
if (tsdbMakeRoom(ppBuf, tsdbBlockStatisSize(pDataCols->numOfCols, SBlockVerLatest)) < 0) {
return -1;
}
pBlockData = (SBlockData *)(*ppBuf);
if (tsdbMakeRoom(ppExBuf, tsdbBlockAggrSize(pDataCols->numOfCols, SBlockVerLatest)) < 0) {
return -1;
}
pAggrBlkData = (SAggrBlkData *)(*ppExBuf);
// Get # of cols not all NULL(not including key column)
int nColsNotAllNull = 0;
for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column
SDataCol *pDataCol = pDataCols->cols + ncol;
SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull;
for (int ncol = 1; ncol < pDataCols->numOfCols; ++ncol) { // ncol from 1, we skip the timestamp column
SDataCol * pDataCol = pDataCols->cols + ncol;
SBlockCol * pBlockCol = pBlockData->cols + nColsNotAllNull;
SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsNotAllNull;
if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it
continue;
}
memset(pBlockCol, 0, sizeof(*pBlockCol));
memset(pAggrBlkCol, 0, sizeof(*pAggrBlkCol));
pBlockCol->colId = pDataCol->colId;
pBlockCol->type = pDataCol->type;
pAggrBlkCol->colId = pDataCol->colId;
if (tDataTypes[pDataCol->type].statisFunc) {
#if 0
(*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max),
&(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex),
&(pBlockCol->numOfNull));
if (pBlockCol->numOfNull == 0) {
#endif
(*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max),
&(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex),
&(pAggrBlkCol->numOfNull));
if (pAggrBlkCol->numOfNull == 0) {
TD_SET_COL_ROWS_NORM(pBlockCol);
} else {
TD_SET_COL_ROWS_MISC(pBlockCol);
......@@ -1181,13 +1268,14 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
// Compress the data if neccessary
int tcol = 0; // counter of not all NULL and written columns
uint32_t toffset = 0;
int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull);
int32_t tsize = (int32_t)tsdbBlockStatisSize(nColsNotAllNull, SBlockVerLatest);
int32_t lsize = tsize;
uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest);
int32_t keyLen = 0;
int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite);
int32_t tBitmaps = 0;
for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) {
// All not NULL columns finish
if (ncol != 0 && tcol >= nColsNotAllNull) break;
......@@ -1248,7 +1336,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
if (ncol != 0) {
tsdbSetBlockColOffset(pBlockCol, toffset);
pBlockCol->len = flen;
tcol++;
++tcol;
} else {
keyLen = flen;
}
......@@ -1269,6 +1357,18 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
return -1;
}
uint32_t aggrStatus = nColsNotAllNull > 0 ? 1 : 0;
if (aggrStatus > 0) {
taosCalcChecksumAppend(0, (uint8_t *)pAggrBlkData, tsizeAggr);
tsdbUpdateDFileMagic(pDFileAggr, POINTER_SHIFT(pAggrBlkData, tsizeAggr - sizeof(TSCKSUM)));
// Write the whole block to file
if (tsdbAppendDFile(pDFileAggr, (void *)pAggrBlkData, tsizeAggr, &offsetAggr) < tsizeAggr) {
return -1;
}
}
// Update pBlock membership vairables
pBlock->last = isLast;
pBlock->offset = offset;
......@@ -1280,6 +1380,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
pBlock->numOfCols = nColsNotAllNull;
pBlock->keyFirst = dataColsKeyFirst(pDataCols);
pBlock->keyLast = dataColsKeyLast(pDataCols);
pBlock->aggrStat = aggrStatus;
pBlock->blkVer = SBlockVerLatest;
pBlock->aggrOffset = (uint64_t)offsetAggr;
tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64
" numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
......@@ -1291,9 +1394,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast,
bool isSuper) {
return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast,
isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))),
(void **)(&(TSDB_COMMIT_COMP_BUF(pCommith))));
return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile,
isLast ? TSDB_COMMIT_SMAL_FILE(pCommith) : TSDB_COMMIT_SMAD_FILE(pCommith), pDataCols,
pBlock, isLast, isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))),
(void **)(&(TSDB_COMMIT_COMP_BUF(pCommith))), (void **)(&(TSDB_COMMIT_EXBUF(pCommith))));
}
static int tsdbWriteBlockInfo(SCommitH *pCommih) {
......
source/dnode/vnode/src/tsdb/tsdbCompact.c
浏览文件 @ f0e392ae
......@@ -38,6 +38,8 @@ typedef struct {
#define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD)
#define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA)
#define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST)
#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD)
#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL)
#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh))
#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh))
......
source/dnode/vnode/src/tsdb/tsdbFS.c
浏览文件 @ f0e392ae
......@@ -422,7 +422,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
return -1;
}
fsheader.version = TSDB_FS_VERSION;
fsheader.version = TSDB_LATEST_SFS_VER;
if (taosArrayGetSize(pStatus->df) == 0) {
fsheader.len = 0;
} else {
......@@ -697,7 +697,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
ptr = tsdbDecodeFSHeader(ptr, &fsheader);
ptr = tsdbDecodeFSMeta(ptr, &(pStatus->meta));
if (fsheader.version != TSDB_FS_VERSION) {
if (fsheader.version != TSDB_LATEST_SFS_VER) {
// TODO: handle file version change
}
......
source/dnode/vnode/src/tsdb/tsdbMain.c
浏览文件 @ f0e392ae
......@@ -821,9 +821,10 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea
// file block with sub-blocks has no statistics data
if (pBlock->numOfSubBlocks <= 1) {
tsdbLoadBlockStatis(pReadh, pBlock);
tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns);
loadStatisData = true;
if (tsdbLoadBlockStatis(pReadh, pBlock) == TSDB_STATIS_OK) {
tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns, pBlock);
loadStatisData = true;
}
}
for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) {
......
source/libs/CMakeLists.txt
浏览文件 @ f0e392ae
......@@ -14,5 +14,6 @@ add_subdirectory(function)
add_subdirectory(qcom)
add_subdirectory(qworker)
add_subdirectory(tfs)
add_subdirectory(monitor)
add_subdirectory(nodes)
add_subdirectory(scalar)
source/libs/cache/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/catalog/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/catalog/src/catalog.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/function/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/inc/indexInt.h
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/inc/index_comm.h 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/src/index.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/src/index_cache.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/src/index_comm.c 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/src/index_json.c 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/src/index_tfile.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/test/fstTest.cc
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/test/fstUT.cc
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/index/test/jsonUT.cc 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/monitor/CMakeLists.txt 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/monitor/inc/monInt.h 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/monitor/src/monitor.c 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/monitor/test/CMakeLists.txt 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/monitor/test/monTest.cpp 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/parser/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/planner/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/qcom/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/qcom/src/querymsg.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/inc/syncEnv.h
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/inc/syncIO.h
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/inc/syncInt.h
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/inc/syncMessage.h
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/inc/syncRaft.h
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/inc/syncUtil.h 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/inc/syncVoteMgr.h
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/src/syncEnv.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/src/syncIO.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/src/syncMain.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/src/syncMessage.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/src/syncRaft.c
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/src/syncUtil.c 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/test/syncEncodeTest.cpp 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/test/syncIOSendMsgClientTest.cpp 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/test/syncIOSendMsgServerTest.cpp 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/test/syncIOSendMsgTest.cpp 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/test/syncIOTickPingTest.cpp 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/test/syncIOTickQTest.cpp 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/test/syncRaftStoreTest.cpp 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/sync/test/syncTest.cpp
浏览文件 @ f0e392ae
此差异已折叠。
source/libs/wal/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/os/CMakeLists.txt
浏览文件 @ f0e392ae
此差异已折叠。
source/os/src/osDir.c
浏览文件 @ f0e392ae
此差异已折叠。
source/os/src/osFile.c
浏览文件 @ f0e392ae
此差异已折叠。
source/os/src/osSysinfo.c
浏览文件 @ f0e392ae
此差异已折叠。
source/util/src/tconfig.c
浏览文件 @ f0e392ae
此差异已折叠。
source/util/src/terror.c
浏览文件 @ f0e392ae
此差异已折叠。
source/util/src/thttp.c 0 → 100644
浏览文件 @ f0e392ae
此差异已折叠。
source/util/src/tjson.c
浏览文件 @ f0e392ae
此差异已折叠。
tools/shell/src/shellMain.c
浏览文件 @ f0e392ae
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或