Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1193
Star 22018
Fork 4786
T
TDengine
“68c8b30e256df133da729e466959dfc091675f47”上不存在“src/kit/git@gitcode.net:qq_37101384/tdengine.git”
提交 f9d7cc44 编写于 作者: H Hongze Cheng
浏览文件
操作

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/vnode

上级 47c3361e bc1d8f6a
隐藏空白更改
内联 并排
Showing with 3110 addition and 1722 deletion
include/common/tmsg.h
浏览文件 @ f9d7cc44
...@@ -140,190 +140,6 @@ typedef enum _mgmt_table { ...@@ -140,190 +140,6 @@ typedef enum _mgmt_table {
#define TSDB_COL_IS_NORMAL_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) #define TSDB_COL_IS_NORMAL_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_NORMAL)
#define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC)
#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0)
typedef struct {
int32_t keyLen;
int32_t valueLen;
void* key;
void* value;
} SKv;
typedef struct {
int32_t connId;
int32_t hbType;
} SClientHbKey;
typedef struct {
SClientHbKey connKey;
SHashObj* info; // hash<Skv.key, Skv>
} SClientHbReq;
typedef struct {
int64_t reqId;
SArray* reqs; // SArray<SClientHbReq>
} SClientHbBatchReq;
typedef struct {
SClientHbKey connKey;
int32_t status;
int32_t bodyLen;
void* body;
} SClientHbRsp;
typedef struct {
int64_t reqId;
int64_t rspId;
SArray* rsps; // SArray<SClientHbRsp>
} SClientHbBatchRsp;
static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) {
return taosIntHash_64(key, keyLen);
}
int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq);
void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq);
int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp);
void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp);
static FORCE_INLINE void tFreeClientHbReq(void *pReq) {
SClientHbReq* req = (SClientHbReq*)pReq;
if (req->info) taosHashCleanup(req->info);
}
int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq);
void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq);
static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) {
SClientHbBatchReq *req = (SClientHbBatchReq*)pReq;
if (deep) {
taosArrayDestroyEx(req->reqs, tFreeClientHbReq);
} else {
taosArrayDestroy(req->reqs);
}
free(pReq);
}
int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp);
void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp);
static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pKv->keyLen);
tlen += taosEncodeFixedI32(buf, pKv->valueLen);
tlen += taosEncodeBinary(buf, pKv->key, pKv->keyLen);
tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen);
return tlen;
}
static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) {
buf = taosDecodeFixedI32(buf, &pKv->keyLen);
buf = taosDecodeFixedI32(buf, &pKv->valueLen);
buf = taosDecodeBinary(buf, &pKv->key, pKv->keyLen);
buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen);
return buf;
}
static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pKey->connId);
tlen += taosEncodeFixedI32(buf, pKey->hbType);
return tlen;
}
static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) {
buf = taosDecodeFixedI32(buf, &pKey->connId);
buf = taosDecodeFixedI32(buf, &pKey->hbType);
return buf;
}
typedef struct SMqHbVgInfo {
int32_t vgId;
} SMqHbVgInfo;
static FORCE_INLINE int taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pVgInfo->vgId);
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqHbVgInfo* pVgInfo) {
buf = taosDecodeFixedI32(buf, &pVgInfo->vgId);
return buf;
}
typedef struct SMqHbTopicInfo {
int32_t epoch;
int64_t topicUid;
char name[TSDB_TOPIC_FNAME_LEN];
SArray* pVgInfo;
} SMqHbTopicInfo;
static FORCE_INLINE int taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch);
tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid);
tlen += taosEncodeString(buf, pTopicInfo->name);
int32_t sz = taosArrayGetSize(pTopicInfo->pVgInfo);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqHbVgInfo* pVgInfo = (SMqHbVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i);
tlen += taosEncodeSMqVgInfo(buf, pVgInfo);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbTopicInfoMsg(void* buf, SMqHbTopicInfo* pTopicInfo) {
buf = taosDecodeFixedI32(buf, &pTopicInfo->epoch);
buf = taosDecodeFixedI64(buf, &pTopicInfo->topicUid);
buf = taosDecodeStringTo(buf, pTopicInfo->name);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqHbVgInfo));
for (int32_t i = 0; i < sz; i++) {
SMqHbVgInfo vgInfo;
buf = taosDecodeSMqVgInfo(buf, &vgInfo);
taosArrayPush(pTopicInfo->pVgInfo, &vgInfo);
}
return buf;
}
typedef struct SMqHbMsg {
int32_t status; // ask hb endpoint
int32_t epoch;
int64_t consumerId;
SArray* pTopics; // SArray<SMqHbTopicInfo>
} SMqHbMsg;
static FORCE_INLINE int taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pMsg->status);
tlen += taosEncodeFixedI32(buf, pMsg->epoch);
tlen += taosEncodeFixedI64(buf, pMsg->consumerId);
int32_t sz = taosArrayGetSize(pMsg->pTopics);
tlen += taosEncodeFixedI32(buf, sz);
for (int i = 0; i < sz; i++) {
SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i);
tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
buf = taosDecodeFixedI32(buf, &pMsg->status);
buf = taosDecodeFixedI32(buf, &pMsg->epoch);
buf = taosDecodeFixedI64(buf, &pMsg->consumerId);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo));
for (int i = 0; i < sz; i++) {
SMqHbTopicInfo topicInfo;
buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo);
taosArrayPush(pMsg->pTopics, &topicInfo);
}
return buf;
}
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
char* dbName; char* dbName;
...@@ -360,6 +176,18 @@ typedef struct SSubmitBlk { ...@@ -360,6 +176,18 @@ typedef struct SSubmitBlk {
char data[]; char data[];
} SSubmitBlk; } SSubmitBlk;
typedef struct {
/* data */
} SSubmitReq;
typedef struct {
/* data */
} SSubmitRsp;
typedef struct {
/* data */
} SSubmitReqReader;
// Submit message for this TSDB // Submit message for this TSDB
typedef struct { typedef struct {
SMsgHead header; SMsgHead header;
...@@ -462,129 +290,43 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { ...@@ -462,129 +290,43 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) {
} }
return buf; return buf;
} }
typedef struct {
int32_t acctId;
int64_t clusterId;
int32_t connId;
int8_t superUser;
int8_t align[3];
SEpSet epSet;
} SConnectRsp;
typedef struct SMqHbRsp { typedef struct {
int8_t status; //idle or not char user[TSDB_USER_LEN];
int8_t vnodeChanged; char pass[TSDB_PASSWORD_LEN];
int8_t epChanged; // should use new epset int32_t maxUsers;
int8_t reserved; int32_t maxDbs;
SEpSet epSet; int32_t maxTimeSeries;
} SMqHbRsp; int32_t maxStreams;
int32_t accessState; // Configured only by command
int64_t maxStorage; // In unit of GB
} SCreateAcctReq, SAlterAcctReq;
static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { typedef struct {
int tlen = 0; char user[TSDB_USER_LEN];
tlen += taosEncodeFixedI8(buf, pRsp->status); } SDropUserReq, SDropAcctReq;
tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged);
tlen += taosEncodeFixedI8(buf, pRsp->epChanged);
tlen += taosEncodeSEpSet(buf, &pRsp->epSet);
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) { typedef struct {
buf = taosDecodeFixedI8(buf, &pRsp->status); int8_t type;
buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged); char user[TSDB_USER_LEN];
buf = taosDecodeFixedI8(buf, &pRsp->epChanged); char pass[TSDB_PASSWORD_LEN];
buf = taosDecodeSEpSet(buf, &pRsp->epSet); int8_t superUser; // denote if it is a super user or not
return buf; } SCreateUserReq, SAlterUserReq;
}
typedef struct SMqHbOneTopicBatchRsp { typedef struct {
char topicName[TSDB_TOPIC_FNAME_LEN]; int16_t colId; // column id
SArray* rsps; // SArray<SMqHbRsp> int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag
} SMqHbOneTopicBatchRsp; int16_t flag; // denote if it is a tag or a normal column
char name[TSDB_DB_FNAME_LEN];
static FORCE_INLINE int taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) { } SColIndex;
int tlen = 0;
tlen += taosEncodeString(buf, pBatchRsp->topicName);
int32_t sz = taosArrayGetSize(pBatchRsp->rsps);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqHbRsp* pRsp = (SMqHbRsp*)taosArrayGet(pBatchRsp->rsps, i);
tlen += taosEncodeSMqHbRsp(buf, pRsp);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTopicBatchRsp* pBatchRsp) {
int32_t sz;
buf = taosDecodeStringTo(buf, pBatchRsp->topicName);
buf = taosDecodeFixedI32(buf, &sz);
pBatchRsp->rsps = taosArrayInit(sz, sizeof(SMqHbRsp));
for (int32_t i = 0; i < sz; i++) {
SMqHbRsp rsp;
buf = taosDecodeSMqHbRsp(buf, &rsp);
buf = taosArrayPush(pBatchRsp->rsps, &rsp);
}
return buf;
}
typedef struct SMqHbBatchRsp {
int64_t consumerId;
SArray* batchRsps; // SArray<SMqHbOneTopicBatchRsp>
} SMqHbBatchRsp;
static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) {
int tlen = 0;
tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId);
int32_t sz;
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*) taosArrayGet(pBatchRsp->batchRsps, i);
tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBatchRsp) {
buf = taosDecodeFixedI64(buf, &pBatchRsp->consumerId);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp));
for (int32_t i = 0; i < sz; i++) {
SMqHbOneTopicBatchRsp rsp;
buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp);
buf = taosArrayPush(pBatchRsp->batchRsps, &rsp);
}
return buf;
}
typedef struct {
int32_t acctId;
int64_t clusterId;
int32_t connId;
int8_t superUser;
int8_t align[3];
SEpSet epSet;
} SConnectRsp;
typedef struct {
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
int32_t maxUsers;
int32_t maxDbs;
int32_t maxTimeSeries;
int32_t maxStreams;
int32_t accessState; // Configured only by command
int64_t maxStorage; // In unit of GB
} SCreateAcctReq, SAlterAcctReq;
typedef struct {
char user[TSDB_USER_LEN];
} SDropUserReq, SDropAcctReq;
typedef struct {
int8_t type;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
int8_t superUser; // denote if it is a super user or not
} SCreateUserReq, SAlterUserReq;
typedef struct {
int16_t colId; // column id
int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag
int16_t flag; // denote if it is a tag or a normal column
char name[TSDB_DB_FNAME_LEN];
} SColIndex;
typedef struct { typedef struct {
int16_t lowerRelOptr; int16_t lowerRelOptr;
...@@ -1128,45 +870,7 @@ typedef struct { ...@@ -1128,45 +870,7 @@ typedef struct {
char desc[TSDB_STEP_DESC_LEN]; char desc[TSDB_STEP_DESC_LEN];
} SStartupReq; } SStartupReq;
// mq related typedef struct SSubQueryMsg {
typedef struct {
} SMqConnectReq;
typedef struct {
} SMqConnectRsp;
typedef struct {
} SMqDisconnectReq;
typedef struct {
} SMqDisconnectRsp;
typedef struct {
} SMqAckReq;
typedef struct {
} SMqAckRsp;
typedef struct {
} SMqResetReq;
typedef struct {
} SMqResetRsp;
// mq related end
typedef struct {
/* data */
} SSubmitReq;
typedef struct {
/* data */
} SSubmitRsp;
typedef struct {
/* data */
} SSubmitReqReader;
typedef struct {
SMsgHead header; SMsgHead header;
uint64_t sId; uint64_t sId;
uint64_t queryId; uint64_t queryId;
...@@ -1385,6 +1089,10 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq ...@@ -1385,6 +1089,10 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq
return buf; return buf;
} }
typedef struct SMqTmrMsg {
int32_t reserved;
} SMqTmrMsg;
typedef struct { typedef struct {
int64_t status; int64_t status;
} SMVSubscribeRsp; } SMVSubscribeRsp;
...@@ -1533,6 +1241,318 @@ typedef struct { ...@@ -1533,6 +1241,318 @@ typedef struct {
#pragma pack(pop) #pragma pack(pop)
static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pMsg->contLen);
tlen += taosEncodeFixedI32(buf, pMsg->vgId);
return tlen;
}
typedef struct SMqHbRsp {
int8_t status; //idle or not
int8_t vnodeChanged;
int8_t epChanged; // should use new epset
int8_t reserved;
SEpSet epSet;
} SMqHbRsp;
static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) {
int tlen = 0;
tlen += taosEncodeFixedI8(buf, pRsp->status);
tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged);
tlen += taosEncodeFixedI8(buf, pRsp->epChanged);
tlen += taosEncodeSEpSet(buf, &pRsp->epSet);
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) {
buf = taosDecodeFixedI8(buf, &pRsp->status);
buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged);
buf = taosDecodeFixedI8(buf, &pRsp->epChanged);
buf = taosDecodeSEpSet(buf, &pRsp->epSet);
return buf;
}
typedef struct SMqHbOneTopicBatchRsp {
char topicName[TSDB_TOPIC_FNAME_LEN];
SArray* rsps; // SArray<SMqHbRsp>
} SMqHbOneTopicBatchRsp;
static FORCE_INLINE int taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) {
int tlen = 0;
tlen += taosEncodeString(buf, pBatchRsp->topicName);
int32_t sz = taosArrayGetSize(pBatchRsp->rsps);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqHbRsp* pRsp = (SMqHbRsp*)taosArrayGet(pBatchRsp->rsps, i);
tlen += taosEncodeSMqHbRsp(buf, pRsp);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTopicBatchRsp* pBatchRsp) {
int32_t sz;
buf = taosDecodeStringTo(buf, pBatchRsp->topicName);
buf = taosDecodeFixedI32(buf, &sz);
pBatchRsp->rsps = taosArrayInit(sz, sizeof(SMqHbRsp));
for (int32_t i = 0; i < sz; i++) {
SMqHbRsp rsp;
buf = taosDecodeSMqHbRsp(buf, &rsp);
buf = taosArrayPush(pBatchRsp->rsps, &rsp);
}
return buf;
}
typedef struct SMqHbBatchRsp {
int64_t consumerId;
SArray* batchRsps; // SArray<SMqHbOneTopicBatchRsp>
} SMqHbBatchRsp;
static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) {
int tlen = 0;
tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId);
int32_t sz;
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*) taosArrayGet(pBatchRsp->batchRsps, i);
tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBatchRsp) {
buf = taosDecodeFixedI64(buf, &pBatchRsp->consumerId);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp));
for (int32_t i = 0; i < sz; i++) {
SMqHbOneTopicBatchRsp rsp;
buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp);
buf = taosArrayPush(pBatchRsp->batchRsps, &rsp);
}
return buf;
}
typedef struct {
int32_t keyLen;
int32_t valueLen;
void* key;
void* value;
} SKv;
typedef struct {
int32_t connId;
int32_t hbType;
} SClientHbKey;
typedef struct {
SClientHbKey connKey;
SHashObj* info; // hash<Skv.key, Skv>
} SClientHbReq;
typedef struct {
int64_t reqId;
SArray* reqs; // SArray<SClientHbReq>
} SClientHbBatchReq;
typedef struct {
SClientHbKey connKey;
int32_t status;
int32_t bodyLen;
void* body;
} SClientHbRsp;
typedef struct {
int64_t reqId;
int64_t rspId;
SArray* rsps; // SArray<SClientHbRsp>
} SClientHbBatchRsp;
static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) {
return taosIntHash_64(key, keyLen);
}
int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq);
void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq);
int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp);
void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp);
static FORCE_INLINE void tFreeClientHbReq(void *pReq) {
SClientHbReq* req = (SClientHbReq*)pReq;
if (req->info) taosHashCleanup(req->info);
}
int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq);
void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq);
static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) {
SClientHbBatchReq *req = (SClientHbBatchReq*)pReq;
if (deep) {
taosArrayDestroyEx(req->reqs, tFreeClientHbReq);
} else {
taosArrayDestroy(req->reqs);
}
free(pReq);
}
int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp);
void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp);
static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pKv->keyLen);
tlen += taosEncodeFixedI32(buf, pKv->valueLen);
tlen += taosEncodeBinary(buf, pKv->key, pKv->keyLen);
tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen);
return tlen;
}
static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) {
buf = taosDecodeFixedI32(buf, &pKv->keyLen);
buf = taosDecodeFixedI32(buf, &pKv->valueLen);
buf = taosDecodeBinary(buf, &pKv->key, pKv->keyLen);
buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen);
return buf;
}
static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pKey->connId);
tlen += taosEncodeFixedI32(buf, pKey->hbType);
return tlen;
}
static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) {
buf = taosDecodeFixedI32(buf, &pKey->connId);
buf = taosDecodeFixedI32(buf, &pKey->hbType);
return buf;
}
typedef struct SMqHbVgInfo {
int32_t vgId;
} SMqHbVgInfo;
static FORCE_INLINE int taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pVgInfo->vgId);
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqHbVgInfo* pVgInfo) {
buf = taosDecodeFixedI32(buf, &pVgInfo->vgId);
return buf;
}
typedef struct SMqHbTopicInfo {
int32_t epoch;
int64_t topicUid;
char name[TSDB_TOPIC_FNAME_LEN];
SArray* pVgInfo;
} SMqHbTopicInfo;
static FORCE_INLINE int taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch);
tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid);
tlen += taosEncodeString(buf, pTopicInfo->name);
int32_t sz = taosArrayGetSize(pTopicInfo->pVgInfo);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqHbVgInfo* pVgInfo = (SMqHbVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i);
tlen += taosEncodeSMqVgInfo(buf, pVgInfo);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbTopicInfoMsg(void* buf, SMqHbTopicInfo* pTopicInfo) {
buf = taosDecodeFixedI32(buf, &pTopicInfo->epoch);
buf = taosDecodeFixedI64(buf, &pTopicInfo->topicUid);
buf = taosDecodeStringTo(buf, pTopicInfo->name);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqHbVgInfo));
for (int32_t i = 0; i < sz; i++) {
SMqHbVgInfo vgInfo;
buf = taosDecodeSMqVgInfo(buf, &vgInfo);
taosArrayPush(pTopicInfo->pVgInfo, &vgInfo);
}
return buf;
}
typedef struct SMqHbMsg {
int32_t status; // ask hb endpoint
int32_t epoch;
int64_t consumerId;
SArray* pTopics; // SArray<SMqHbTopicInfo>
} SMqHbMsg;
static FORCE_INLINE int taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, pMsg->status);
tlen += taosEncodeFixedI32(buf, pMsg->epoch);
tlen += taosEncodeFixedI64(buf, pMsg->consumerId);
int32_t sz = taosArrayGetSize(pMsg->pTopics);
tlen += taosEncodeFixedI32(buf, sz);
for (int i = 0; i < sz; i++) {
SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i);
tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
buf = taosDecodeFixedI32(buf, &pMsg->status);
buf = taosDecodeFixedI32(buf, &pMsg->epoch);
buf = taosDecodeFixedI64(buf, &pMsg->consumerId);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo));
for (int i = 0; i < sz; i++) {
SMqHbTopicInfo topicInfo;
buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo);
taosArrayPush(pMsg->pTopics, &topicInfo);
}
return buf;
}
typedef struct SMqSetCVgReq {
int32_t vgId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cGroup[TSDB_CONSUMER_GROUP_LEN];
char* sql;
char* logicalPlan;
char* physicalPlan;
SArray* tasks; // SArray<SSubQueryMsg>
} SMqSetCVgReq;
static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeString(buf, pReq->topicName);
tlen += taosEncodeString(buf, pReq->cGroup);
tlen += taosEncodeString(buf, pReq->sql);
tlen += taosEncodeString(buf, pReq->logicalPlan);
tlen += taosEncodeString(buf, pReq->physicalPlan);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->topicName);
buf = taosDecodeStringTo(buf, pReq->cGroup);
buf = taosDecodeString(buf, &pReq->sql);
buf = taosDecodeString(buf, &pReq->logicalPlan);
buf = taosDecodeString(buf, &pReq->physicalPlan);
pReq->tasks = NULL;
return buf;
}
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
include/common/tmsgdef.h
浏览文件 @ f9d7cc44
...@@ -140,6 +140,7 @@ enum { ...@@ -140,6 +140,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-timer", SMqTmrMsg, SMqTmrMsg)
// Requests handled by VNODE // Requests handled by VNODE
TD_NEW_MSG_SEG(TDMT_VND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG)
......
include/dnode/mnode/sdb/sdb.h
浏览文件 @ f9d7cc44
...@@ -113,8 +113,8 @@ typedef enum { ...@@ -113,8 +113,8 @@ typedef enum {
SDB_USER = 7, SDB_USER = 7,
SDB_AUTH = 8, SDB_AUTH = 8,
SDB_ACCT = 9, SDB_ACCT = 9,
SDB_CONSUMER = 10, SDB_SUBSCRIBE = 10,
SDB_CGROUP = 11, SDB_CONSUMER = 11,
SDB_TOPIC = 12, SDB_TOPIC = 12,
SDB_VGROUP = 13, SDB_VGROUP = 13,
SDB_STB = 14, SDB_STB = 14,
......
include/libs/scheduler/scheduler.h
浏览文件 @ f9d7cc44
...@@ -114,6 +114,14 @@ void schedulerDestroy(void); ...@@ -114,6 +114,14 @@ void schedulerDestroy(void);
*/ */
int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks); int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks);
/**
* make one task info's multiple copies
* @param src
* @param dst SArray**<STaskInfo>
* @return
*/
int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum);
void schedulerFreeTaskList(SArray *taskList); void schedulerFreeTaskList(SArray *taskList);
......
include/libs/tfs/tfs.h
浏览文件 @ f9d7cc44
...@@ -16,77 +16,226 @@ ...@@ -16,77 +16,226 @@
#ifndef _TD_TFS_H_ #ifndef _TD_TFS_H_
#define _TD_TFS_H_ #define _TD_TFS_H_
#include "tglobal.h" #include "tcfg.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
/* ------------------------ TYPES EXPOSED ------------------------ */
typedef struct STfs STfs;
typedef struct STfsDir STfsDir;
typedef struct { typedef struct {
int32_t level; int32_t level;
int32_t id; int32_t id;
} SDiskID; } SDiskID;
#define TFS_UNDECIDED_LEVEL -1
#define TFS_UNDECIDED_ID -1
#define TFS_PRIMARY_LEVEL 0
#define TFS_PRIMARY_ID 0
#define TFS_MIN_LEVEL 0
#define TFS_MAX_LEVEL (TSDB_MAX_TIERS - 1)
// FS APIs ====================================
typedef struct { typedef struct {
int64_t total; SDiskID did;
int64_t used; char aname[TSDB_FILENAME_LEN]; // ABS name
int64_t avail; char rname[TSDB_FILENAME_LEN]; // REL name
} SFSMeta; STfs *pTfs;
} STfsFile;
int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk); /**
void tfsCleanup(); * @brief Open a fs.
void tfsUpdateSize(SFSMeta *pFSMeta); *
void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id); * @param pCfg Config of the fs.
* @param ndisk Length of the config.
* @return STfs* The fs object.
*/
STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk);
const char *TFS_PRIMARY_PATH(); /**
const char *TFS_DISK_PATH(int32_t level, int32_t id); * @brief Close a fs.
*
* @param pTfs The fs object to close.
*/
void tfsClose(STfs *pTfs);
// TFILE APIs ==================================== /**
typedef struct { * @brief Update the disk size.
int32_t level; *
int32_t id; * @param pTfs The fs object.
char rname[TSDB_FILENAME_LEN]; // REL name */
char aname[TSDB_FILENAME_LEN]; // ABS name void tfsUpdateSize(STfs *pTfs);
} TFILE;
/**
#define TFILE_LEVEL(pf) ((pf)->level) * @brief Get the disk size.
#define TFILE_ID(pf) ((pf)->id) *
#define TFILE_NAME(pf) ((pf)->aname) * @param pTfs The fs object.
#define TFILE_REL_NAME(pf) ((pf)->rname) */
SDiskSize tfsGetSize(STfs *pTfs);
#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
#define tfsclose(fd) close(fd) /**
#define tfsremove(pf) remove(TFILE_NAME(pf)) * @brief Allocate an existing available tier level from fs.
#define tfscopy(sf, df) taosCopyFile(TFILE_NAME(sf), TFILE_NAME(df)) *
#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df)) * @param pTfs The fs object.
* @param expLevel Disk level want to allocate.
void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname); * @param pDiskId The disk ID after allocation.
bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2); * @return int32_t 0 for success, -1 for failure.
int32_t tfsEncodeFile(void **buf, TFILE *pf); */
void *tfsDecodeFile(void *buf, TFILE *pf); int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId);
void tfsbasename(const TFILE *pf, char *dest);
void tfsdirname(const TFILE *pf, char *dest); /**
* @brief Get the primary path.
// DIR APIs ==================================== *
int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id); * @param pTfs The fs object.
int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id); * @return const char * The primary path.
int32_t tfsMkdir(const char *rname); */
int32_t tfsRmdir(const char *rname); const char *tfsGetPrimaryPath(STfs *pTfs);
int32_t tfsRename(char *orname, char *nrname);
/**
typedef struct TDIR TDIR; * @brief Get the disk path.
*
TDIR *tfsOpendir(const char *rname); * @param pTfs The fs object.
const TFILE *tfsReaddir(TDIR *tdir); * @param diskId The diskId.
void tfsClosedir(TDIR *tdir); * @return const char * The primary path.
*/
const char *tfsGetDiskPath(STfs *pTfs, SDiskID diskId);
/**
* @brief Make directory at all levels in tfs.
*
* @param pTfs The fs object.
* @param rname The rel name of directory.
* @return int32_t 0 for success, -1 for failure.
*/
int32_t tfsMkdir(STfs *pTfs, const char *rname);
/**
* @brief Create directories in tfs.
*
* @param pTfs The fs object.
* @param rname The rel name of directory.
* @param diskId The disk ID.
* @return int32_t 0 for success, -1 for failure.
*/
int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId);
/**
* @brief Recursive create directories in tfs.
*
* @param pTfs The fs object.
* @param rname The rel name of directory.
* @param diskId The disk ID.
* @return int32_t 0 for success, -1 for failure.
*/
int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId);
/**
* @brief Remove directory at all levels in tfs.
*
* @param pTfs The fs object.
* @param rname The rel name of directory.
* @return int32_t 0 for success, -1 for failure.
*/
int32_t tfsRmdir(STfs *pTfs, const char *rname);
/**
* @brief Rename file/directory in tfs.
*
* @param pTfs The fs object.
* @param orname The rel name of old file.
* @param nrname The rel name of new file.
* @return int32_t 0 for success, -1 for failure.
*/
int32_t tfsRename(STfs *pTfs, char *orname, char *nrname);
/**
* @brief Init file object in tfs.
*
* @param pTfs The fs object.
* @param pFile The file object.
* @param diskId The disk ID.
* @param rname The rel name of file.
*/
void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname);
/**
* @brief Determine whether they are the same file.
*
* @param pFile1 The file object.
* @param pFile2 The file object.
* @param bool The compare result.
*/
bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2);
/**
* @brief Encode file name to a buffer.
*
* @param buf The buffer where file name are saved.
* @param pFile The file object.
* @return int32_t 0 for success, -1 for failure.
*/
int32_t tfsEncodeFile(void **buf, STfsFile *pFile);
/**
* @brief Decode file name from a buffer.
*
* @param pTfs The fs object.
* @param buf The buffer where file name are saved.
* @param pFile The file object.
* @return void * Buffer address after decode.
*/
void *tfsDecodeFile(STfs *pTfs, void *buf, STfsFile *pFile);
/**
* @brief Get the basename of the file.
*
* @param pFile The file object.
* @param dest The buffer where basename will be saved.
*/
void tfsBasename(const STfsFile *pFile, char *dest);
/**
* @brief Get the dirname of the file.
*
* @param pFile The file object.
* @param dest The buffer where dirname will be saved.
*/
void tfsDirname(const STfsFile *pFile, char *dest);
/**
* @brief Remove file in tfs.
*
* @param pFile The file to be removed.
* @return int32_t 0 for success, -1 for failure.
*/
int32_t tfsRemoveFile(const STfsFile *pFile);
/**
* @brief Copy file in tfs.
*
* @param pFile1 The src file.
* @param pFile2 The dest file.
* @return int32_t 0 for success, -1 for failure.
*/
int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2);
/**
* @brief Open a directory for traversal.
*
* @param rname The rel name of file.
* @return STfsDir* The dir object.
*/
STfsDir *tfsOpendir(STfs *pTfs, const char *rname);
/**
* @brief Get a file from dir and move to next pos.
*
* @param pDir The dir object.
* @return STfsFile* The file in dir.
*/
const STfsFile *tfsReaddir(STfsDir *pDir);
/**
* @brief Close a directory.
*
* @param pDir The dir object.
*/
void tfsClosedir(STfsDir *pDir);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
include/os/osFile.h
浏览文件 @ f9d7cc44
...@@ -58,8 +58,8 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); ...@@ -58,8 +58,8 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count);
void taosCloseFile(FileFd fd); void taosCloseFile(FileFd fd);
int32_t taosRenameFile(char *oldName, char *newName); int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(char *from, char *to); int64_t taosCopyFile(const char *from, const char *to);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
......
include/util/taoserror.h
浏览文件 @ f9d7cc44
...@@ -361,6 +361,7 @@ int32_t* taosGetErrno(); ...@@ -361,6 +361,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_TASK_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0717) //"Task dropping") #define TSDB_CODE_QRY_TASK_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0717) //"Task dropping")
#define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) //"Duplicatted operation") #define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) //"Duplicatted operation")
#define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) //"Task message error") #define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) //"Task message error")
#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A) //"Job freed")
// grant // grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired")
......
include/util/tdef.h
浏览文件 @ f9d7cc44
...@@ -178,6 +178,7 @@ do { \ ...@@ -178,6 +178,7 @@ do { \
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN
#define TSDB_CONSUMER_GROUP_LEN 192 #define TSDB_CONSUMER_GROUP_LEN 192
#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
#define TSDB_COL_NAME_LEN 65 #define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE
...@@ -372,10 +373,14 @@ do { \ ...@@ -372,10 +373,14 @@ do { \
#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P #define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P
#define TSDB_MAX_TIERS 3 #define TFS_MAX_TIERS 3
#define TSDB_MAX_DISKS_PER_TIER 16 #define TFS_MAX_DISKS_PER_TIER 16
#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER) #define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER)
#define TFS_MIN_LEVEL 0
#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1)
#define TFS_PRIMARY_LEVEL 0
#define TFS_PRIMARY_ID 0
#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED }; enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED };
enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK }; enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK };
......
source/client/test/clientTests.cpp
浏览文件 @ f9d7cc44
...@@ -148,30 +148,30 @@ TEST(testCase, connect_Test) { ...@@ -148,30 +148,30 @@ TEST(testCase, connect_Test) {
// taos_close(pConn); // taos_close(pConn);
//} //}
// //
//TEST(testCase, create_db_Test) { TEST(testCase, create_db_Test) {
//TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
//assert(pConn != NULL); assert(pConn != NULL);
//TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
//if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
//printf("error in create db, reason:%s\n", taos_errstr(pRes)); printf("error in create db, reason:%s\n", taos_errstr(pRes));
//} }
//TAOS_FIELD* pFields = taos_fetch_fields(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes);
//ASSERT_TRUE(pFields == NULL); ASSERT_TRUE(pFields == NULL);
//int32_t numOfFields = taos_num_fields(pRes); int32_t numOfFields = taos_num_fields(pRes);
//ASSERT_EQ(numOfFields, 0); ASSERT_EQ(numOfFields, 0);
//taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create database abc1 vgroups 4");
if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes));
}
taos_close(pConn);
}
//pRes = taos_query(pConn, "create database abc1 vgroups 4");
//if (taos_errno(pRes) != 0) {
//printf("error in create db, reason:%s\n", taos_errstr(pRes));
//}
//taos_close(pConn);
//}
//
//TEST(testCase, create_dnode_Test) { //TEST(testCase, create_dnode_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); // assert(pConn != NULL);
......
source/common/src/tglobal.c
浏览文件 @ f9d7cc44
...@@ -137,7 +137,7 @@ int32_t tsDiskCfgNum = 0; ...@@ -137,7 +137,7 @@ int32_t tsDiskCfgNum = 0;
#ifndef _STORAGE #ifndef _STORAGE
SDiskCfg tsDiskCfg[1]; SDiskCfg tsDiskCfg[1];
#else #else
SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; SDiskCfg tsDiskCfg[TFS_MAX_DISKS];
#endif #endif
/* /*
......
source/dnode/mgmt/impl/inc/dndEnv.h
浏览文件 @ f9d7cc44
...@@ -134,6 +134,7 @@ typedef struct SDnode { ...@@ -134,6 +134,7 @@ typedef struct SDnode {
SBnodeMgmt bmgmt; SBnodeMgmt bmgmt;
SVnodesMgmt vmgmt; SVnodesMgmt vmgmt;
STransMgmt tmgmt; STransMgmt tmgmt;
STfs *pTfs;
SStartupReq startup; SStartupReq startup;
} SDnode; } SDnode;
......
source/dnode/mgmt/impl/inc/dndInt.h
浏览文件 @ f9d7cc44
...@@ -43,6 +43,7 @@ extern "C" { ...@@ -43,6 +43,7 @@ extern "C" {
#include "qnode.h" #include "qnode.h"
#include "snode.h" #include "snode.h"
#include "vnode.h" #include "vnode.h"
#include "tfs.h"
extern int32_t dDebugFlag; extern int32_t dDebugFlag;
......
source/dnode/mgmt/impl/src/dndEnv.c
浏览文件 @ f9d7cc44
...@@ -173,11 +173,12 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) { ...@@ -173,11 +173,12 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) {
return NULL; return NULL;
} }
SDiskCfg dCfg; SDiskCfg dCfg = {0};
strcpy(dCfg.dir, pDnode->cfg.dataDir); tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN);
dCfg.level = 0; dCfg.level = 0;
dCfg.primary = 1; dCfg.primary = 1;
if (tfsInit(&dCfg, 1) != 0) { pDnode->pTfs = tfsOpen(&dCfg, 1);
if (pDnode->pTfs == NULL) {
dError("failed to init tfs since %s", terrstr()); dError("failed to init tfs since %s", terrstr());
dndClose(pDnode); dndClose(pDnode);
return NULL; return NULL;
...@@ -251,7 +252,7 @@ void dndClose(SDnode *pDnode) { ...@@ -251,7 +252,7 @@ void dndClose(SDnode *pDnode) {
dndCleanupQnode(pDnode); dndCleanupQnode(pDnode);
dndCleanupVnodes(pDnode); dndCleanupVnodes(pDnode);
dndCleanupMgmt(pDnode); dndCleanupMgmt(pDnode);
tfsCleanup(); tfsClose(pDnode->pTfs);
dndCloseImp(pDnode); dndCloseImp(pDnode);
free(pDnode); free(pDnode);
...@@ -313,4 +314,28 @@ void dndCleanup() { ...@@ -313,4 +314,28 @@ void dndCleanup() {
taosStopCacheRefreshWorker(); taosStopCacheRefreshWorker();
dInfo("dnode env is cleaned up"); dInfo("dnode env is cleaned up");
}
// OTHER FUNCTIONS ===================================
void taosGetDisk() {
#if 0
const double unit = 1024 * 1024 * 1024;
SDiskSize diskSize = tfsGetSize(pTfs);
tfsUpdateSize(&fsMeta);
tsTotalDataDirGB = (float)(fsMeta.total / unit);
tsUsedDataDirGB = (float)(fsMeta.used / unit);
tsAvailDataDirGB = (float)(fsMeta.avail / unit);
if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
tsTotalLogDirGB = (float)(diskSize.total / unit);
tsAvailLogDirGB = (float)(diskSize.avail / unit);
}
if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
tsTotalTmpDirGB = (float)(diskSize.total / unit);
tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
}
#endif
} }
\ No newline at end of file
source/dnode/mgmt/impl/src/dndTransport.c
浏览文件 @ f9d7cc44
...@@ -100,7 +100,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { ...@@ -100,7 +100,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg;
......
source/dnode/mgmt/impl/src/dndVnodes.c
浏览文件 @ f9d7cc44
...@@ -381,7 +381,7 @@ static void *dnodeOpenVnodeFunc(void *param) { ...@@ -381,7 +381,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
pMgmt->openVnodes, pMgmt->totalVnodes); pMgmt->openVnodes, pMgmt->totalVnodes);
dndReportStartup(pDnode, "open-vnodes", stepDesc); dndReportStartup(pDnode, "open-vnodes", stepDesc);
SVnodeCfg cfg = {.pDnode = pDnode, .vgId = pCfg->vgId}; SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId};
SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); SVnode *pImpl = vnodeOpen(pCfg->path, &cfg);
if (pImpl == NULL) { if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
...@@ -587,6 +587,7 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { ...@@ -587,6 +587,7 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
} }
vnodeCfg.pDnode = pDnode; vnodeCfg.pDnode = pDnode;
vnodeCfg.pTfs = pDnode->pTfs;
SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg);
if (pImpl == NULL) { if (pImpl == NULL) {
dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr());
...@@ -841,6 +842,7 @@ static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { ...@@ -841,6 +842,7 @@ static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) {
SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
dError("vgId:%d, failed to acquire vnode while process req", pHead->vgId);
if (pMsg->msgType & 1u) { if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
rpcSendResponse(&rsp); rpcSendResponse(&rsp);
......
source/dnode/mnode/impl/CMakeLists.txt
浏览文件 @ f9d7cc44
...@@ -7,6 +7,7 @@ target_include_directories( ...@@ -7,6 +7,7 @@ target_include_directories(
) )
target_link_libraries( target_link_libraries(
mnode mnode
PRIVATE scheduler
PRIVATE sdb PRIVATE sdb
PRIVATE wal PRIVATE wal
PRIVATE transport PRIVATE transport
...@@ -16,4 +17,4 @@ target_link_libraries( ...@@ -16,4 +17,4 @@ target_link_libraries(
if(${BUILD_TEST}) if(${BUILD_TEST})
add_subdirectory(test) add_subdirectory(test)
endif(${BUILD_TEST}) endif(${BUILD_TEST})
\ No newline at end of file
source/dnode/mnode/impl/inc/mndConsumer.h
浏览文件 @ f9d7cc44
...@@ -28,6 +28,9 @@ void mndCleanupConsumer(SMnode *pMnode); ...@@ -28,6 +28,9 @@ void mndCleanupConsumer(SMnode *pMnode);
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId); SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId);
void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ f9d7cc44
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
#include "tlog.h" #include "tlog.h"
#include "trpc.h" #include "trpc.h"
#include "ttimer.h" #include "ttimer.h"
#include "scheduler.h"
#include "mnode.h" #include "mnode.h"
...@@ -326,17 +327,157 @@ typedef struct SMqTopicConsumer { ...@@ -326,17 +327,157 @@ typedef struct SMqTopicConsumer {
#endif #endif
typedef struct SMqConsumerEp { typedef struct SMqConsumerEp {
int32_t vgId; int32_t vgId; // -1 for unassigned
SEpSet epset; SEpSet epset;
int64_t consumerId; int64_t consumerId; // -1 for unassigned
int64_t lastConsumerHbTs;
int64_t lastVgHbTs;
} SMqConsumerEp; } SMqConsumerEp;
typedef struct SMqCgroupTopicPair { static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
char key[TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN]; int32_t tlen = 0;
SArray* assigned; // SArray<SMqConsumerEp> tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId);
SArray* unassignedConsumer; tlen += taosEncodeSEpSet(buf, &pConsumerEp->epset);
SArray* unassignedVg; tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
} SMqCgroupTopicPair; return tlen;
}
static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId);
buf = taosDecodeSEpSet(buf, &pConsumerEp->epset);
buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
return buf;
}
//unit for rebalance
typedef struct SMqSubscribeObj {
char key[TSDB_SUBSCRIBE_KEY_LEN];
int32_t epoch;
//TODO: replace with priority queue
int32_t nextConsumerIdx;
SArray* availConsumer; // SArray<int64_t> (consumerId)
SArray* assigned; // SArray<SMqConsumerEp>
SArray* unassignedConsumer; // SArray<SMqConsumerEp>
SArray* unassignedVg; // SArray<SMqConsumerEp>
} SMqSubscribeObj;
static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
SMqSubscribeObj* pSub = malloc(sizeof(SMqSubscribeObj));
pSub->key[0] = 0;
pSub->epoch = 0;
if (pSub == NULL) {
return NULL;
}
pSub->availConsumer = taosArrayInit(0, sizeof(int64_t));
if (pSub->availConsumer == NULL) {
free(pSub);
return NULL;
}
pSub->assigned = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->assigned == NULL) {
taosArrayDestroy(pSub->availConsumer);
free(pSub);
return NULL;
}
pSub->unassignedConsumer = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->assigned == NULL) {
taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->unassignedConsumer);
free(pSub);
return NULL;
}
pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->assigned == NULL) {
taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->unassignedConsumer);
taosArrayDestroy(pSub->unassignedVg);
free(pSub);
return NULL;
}
return NULL;
}
static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pSub->key);
tlen += taosEncodeFixedI32(buf, pSub->epoch);
int32_t sz;
sz = taosArrayGetSize(pSub->availConsumer);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
int64_t* pConsumerId = taosArrayGet(pSub->availConsumer, i);
tlen += taosEncodeFixedI64(buf, *pConsumerId);
}
sz = taosArrayGetSize(pSub->assigned);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp* pCEp = taosArrayGet(pSub->assigned, i);
tlen += tEncodeSMqConsumerEp(buf, pCEp);
}
sz = taosArrayGetSize(pSub->unassignedConsumer);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedConsumer, i);
tlen += tEncodeSMqConsumerEp(buf, pCEp);
}
sz = taosArrayGetSize(pSub->unassignedVg);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedVg, i);
tlen += tEncodeSMqConsumerEp(buf, pCEp);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) {
buf = taosDecodeStringTo(buf, pSub->key);
buf = taosDecodeFixedI32(buf, &pSub->epoch);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pSub->assigned = taosArrayInit(sz, sizeof(int64_t));
if (pSub->assigned == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
int64_t consumerId;
buf = taosDecodeFixedI64(buf, &consumerId);
taosArrayPush(pSub->assigned, &consumerId);
}
buf = taosDecodeFixedI32(buf, &sz);
pSub->unassignedConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->unassignedConsumer == NULL) {
taosArrayDestroy(pSub->assigned);
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp cEp;
buf = tDecodeSMqConsumerEp(buf, &cEp);
taosArrayPush(pSub->unassignedConsumer, &cEp);
}
buf = taosDecodeFixedI32(buf, &sz);
pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->unassignedVg == NULL) {
taosArrayDestroy(pSub->assigned);
taosArrayDestroy(pSub->unassignedConsumer);
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp cEp;
buf = tDecodeSMqConsumerEp(buf, &cEp);
taosArrayPush(pSub->unassignedVg, &cEp);
}
return buf;
}
typedef struct SMqCGroup { typedef struct SMqCGroup {
char name[TSDB_CONSUMER_GROUP_LEN]; char name[TSDB_CONSUMER_GROUP_LEN];
...@@ -358,8 +499,8 @@ typedef struct SMqTopicObj { ...@@ -358,8 +499,8 @@ typedef struct SMqTopicObj {
char *sql; char *sql;
char *logicalPlan; char *logicalPlan;
char *physicalPlan; char *physicalPlan;
SHashObj *cgroups; // SHashObj<SMqCGroup> //SHashObj *cgroups; // SHashObj<SMqCGroup>
SHashObj *consumers; // SHashObj<SMqConsumerObj> //SHashObj *consumers; // SHashObj<SMqConsumerObj>
} SMqTopicObj; } SMqTopicObj;
// TODO: add cache and change name to id // TODO: add cache and change name to id
...@@ -367,18 +508,93 @@ typedef struct SMqConsumerTopic { ...@@ -367,18 +508,93 @@ typedef struct SMqConsumerTopic {
char name[TSDB_TOPIC_FNAME_LEN]; char name[TSDB_TOPIC_FNAME_LEN];
int32_t epoch; int32_t epoch;
//TODO: replace with something with ep //TODO: replace with something with ep
SList *vgroups; // SList<int32_t> //SList *vgroups; // SList<int32_t>
//vg assigned to the consumer on the topic
SArray *pVgInfo; // SArray<int32_t> SArray *pVgInfo; // SArray<int32_t>
} SMqConsumerTopic; } SMqConsumerTopic;
static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic));
if (pCTopic == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
strcpy(pCTopic->name, pTopic->name);
pCTopic->epoch = 0;
pCTopic->pVgInfo = taosArrayInit(0, sizeof(int32_t));
int32_t unassignedVgSz = taosArrayGetSize(pSub->unassignedVg);
if (unassignedVgSz > 0) {
SMqConsumerEp* pCEp = taosArrayPop(pSub->unassignedVg);
pCEp->consumerId = consumerId;
taosArrayPush(pCTopic->pVgInfo, &pCEp->vgId);
taosArrayPush(pSub->assigned, pCEp);
}
return pCTopic;
}
static FORCE_INLINE int32_t tEncodeSMqConsumerTopic(void** buf, SMqConsumerTopic* pConsumerTopic) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pConsumerTopic->name);
tlen += taosEncodeFixedI32(buf, pConsumerTopic->epoch);
int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
int32_t* pVgInfo = taosArrayGet(pConsumerTopic->pVgInfo, i);
tlen += taosEncodeFixedI32(buf, *pVgInfo);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSMqConsumerTopic(void* buf, SMqConsumerTopic* pConsumerTopic) {
buf = taosDecodeStringTo(buf, pConsumerTopic->name);
buf = taosDecodeFixedI32(buf, &pConsumerTopic->epoch);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pConsumerTopic->pVgInfo = taosArrayInit(sz, sizeof(SMqConsumerTopic));
for (int32_t i = 0; i < sz; i++) {
int32_t vgInfo;
buf = taosDecodeFixedI32(buf, &vgInfo);
taosArrayPush(pConsumerTopic->pVgInfo, &vgInfo);
}
return buf;
}
typedef struct SMqConsumerObj { typedef struct SMqConsumerObj {
int64_t consumerId; int64_t consumerId;
SRWLatch lock; SRWLatch lock;
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN];
SArray *topics; // SArray<SMqConsumerTopic> SArray *topics; // SArray<SMqConsumerTopic>
SHashObj *topicHash; //SHashObj<SMqConsumerTopic> //SHashObj *topicHash; //SHashObj<SMqTopicObj>
} SMqConsumerObj; } SMqConsumerObj;
static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
tlen += taosEncodeString(buf, pConsumer->cgroup);
int32_t sz = taosArrayGetSize(pConsumer->topics);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqConsumerTopic* pConsumerTopic = taosArrayGet(pConsumer->topics, i);
tlen += tEncodeSMqConsumerTopic(buf, pConsumerTopic);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) {
buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
buf = taosDecodeStringTo(buf, pConsumer->cgroup);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pConsumer->topics = taosArrayInit(sz, sizeof(SMqConsumerObj));
for (int32_t i = 0; i < sz; i++) {
SMqConsumerTopic cTopic;
buf = tDecodeSMqConsumerTopic(buf, &cTopic);
taosArrayPush(pConsumer->topics, &cTopic);
}
return buf;
}
typedef struct SMqSubConsumerObj { typedef struct SMqSubConsumerObj {
int64_t consumerUid; // if -1, unassigned int64_t consumerUid; // if -1, unassigned
SList *vgId; // SList<int32_t> SList *vgId; // SList<int32_t>
......
source/dnode/mnode/impl/inc/mndInt.h
浏览文件 @ f9d7cc44
...@@ -80,6 +80,7 @@ typedef struct SMnode { ...@@ -80,6 +80,7 @@ typedef struct SMnode {
SReplica replicas[TSDB_MAX_REPLICA]; SReplica replicas[TSDB_MAX_REPLICA];
tmr_h timer; tmr_h timer;
tmr_h transTimer; tmr_h transTimer;
tmr_h mqTimer;
char *path; char *path;
SMnodeCfg cfg; SMnodeCfg cfg;
int64_t checkTime; int64_t checkTime;
......
source/dnode/mnode/impl/inc/mndSubscribe.h 0 → 100644
浏览文件 @ f9d7cc44
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_SUBSCRIBE_H_
#define _TD_MND_SUBSCRIBE_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitSubscribe(SMnode *pMnode);
void mndCleanupSubscribe(SMnode *pMnode);
SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, char *CGroup, char *topicName);
void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
SSdbRaw *mndSubscribeActionEncode(SMqSubscribeObj *pSub);
SSdbRow *mndSubscribeActionDecode(SSdbRaw *pRaw);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_SUBSCRIBE_H_*/
source/dnode/mnode/impl/src/mndAuth.c
浏览文件 @ f9d7cc44
...@@ -16,9 +16,18 @@ ...@@ -16,9 +16,18 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndAuth.h" #include "mndAuth.h"
int32_t mndInitAuth(SMnode *pMnode) { return 0; } static int32_t mndProcessAuthReq(SMnodeMsg *pReq);
void mndCleanupAuth(SMnode *pMnode) {}
int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { int32_t mndInitAuth(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq);
return 0;
}
void mndCleanupAuth(SMnode *pMnode) {}
int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; }
static int32_t mndProcessAuthReq(SMnodeMsg *pReq) {
mDebug("user:%s, auth req is processed", pReq->user);
return 0; return 0;
} }
\ No newline at end of file
source/dnode/mnode/impl/src/mndConsumer.c
浏览文件 @ f9d7cc44
...@@ -30,24 +30,14 @@ ...@@ -30,24 +30,14 @@
#define MND_CONSUMER_VER_NUMBER 1 #define MND_CONSUMER_VER_NUMBER 1
#define MND_CONSUMER_RESERVE_SIZE 64 #define MND_CONSUMER_RESERVE_SIZE 64
static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
static int32_t mndProcessCreateConsumerMsg(SMnodeMsg *pMsg);
static int32_t mndProcessDropConsumerMsg(SMnodeMsg *pMsg);
static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg);
static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg);
static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta);
static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter);
static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg);
static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg);
static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg);
static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg);
int32_t mndInitConsumer(SMnode *pMnode) { int32_t mndInitConsumer(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_CONSUMER, SSdbTable table = {.sdbType = SDB_CONSUMER,
.keyType = SDB_KEY_BINARY, .keyType = SDB_KEY_BINARY,
...@@ -57,26 +47,29 @@ int32_t mndInitConsumer(SMnode *pMnode) { ...@@ -57,26 +47,29 @@ int32_t mndInitConsumer(SMnode *pMnode) {
.updateFp = (SdbUpdateFp)mndConsumerActionUpdate, .updateFp = (SdbUpdateFp)mndConsumerActionUpdate,
.deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; .deleteFp = (SdbDeleteFp)mndConsumerActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
/*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/
/*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/
mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp);
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
} }
void mndCleanupConsumer(SMnode *pMnode) {} void mndCleanupConsumer(SMnode *pMnode) {}
static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) { SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
return 0; terrno = TSDB_CODE_OUT_OF_MEMORY;
} int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer);
SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, tlen);
static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
int32_t size = sizeof(SMqConsumerObj) + MND_CONSUMER_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size);
if (pRaw == NULL) goto CM_ENCODE_OVER; if (pRaw == NULL) goto CM_ENCODE_OVER;
void* buf = malloc(tlen);
if (buf == NULL) goto CM_ENCODE_OVER;
void* abuf = buf;
tEncodeSMqConsumerObj(&abuf, pConsumer);
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, CM_ENCODE_OVER);
#if 0
int32_t topicNum = taosArrayGetSize(pConsumer->topics); int32_t topicNum = taosArrayGetSize(pConsumer->topics);
SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId, CM_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId, CM_ENCODE_OVER);
int32_t len = strlen(pConsumer->cgroup); int32_t len = strlen(pConsumer->cgroup);
...@@ -101,10 +94,13 @@ static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { ...@@ -101,10 +94,13 @@ static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
/*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/ /*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/
} }
} }
#endif
SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER); SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER);
terrno = TSDB_CODE_SUCCESS;
CM_ENCODE_OVER: CM_ENCODE_OVER:
if (terrno != 0) { if (terrno != 0) {
mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr());
...@@ -116,7 +112,7 @@ CM_ENCODE_OVER: ...@@ -116,7 +112,7 @@ CM_ENCODE_OVER:
return pRaw; return pRaw;
} }
static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
int8_t sver = 0; int8_t sver = 0;
...@@ -127,18 +123,27 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { ...@@ -127,18 +123,27 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
goto CONSUME_DECODE_OVER; goto CONSUME_DECODE_OVER;
} }
int32_t size = sizeof(SMqConsumerObj); SSdbRow *pRow = sdbAllocRow(sizeof(SMqConsumerObj));
SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto CONSUME_DECODE_OVER; if (pRow == NULL) goto CONSUME_DECODE_OVER;
SMqConsumerObj *pConsumer = sdbGetRowObj(pRow); SMqConsumerObj *pConsumer = sdbGetRowObj(pRow);
if (pConsumer == NULL) goto CONSUME_DECODE_OVER; if (pConsumer == NULL) goto CONSUME_DECODE_OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_GET_INT64(pRaw, dataPos, &pConsumer->consumerId, CONSUME_DECODE_OVER); int32_t len;
int32_t len, topicNum;
SDB_GET_INT32(pRaw, dataPos, &len, CONSUME_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, CONSUME_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CONSUME_DECODE_OVER); void* buf = malloc(len);
if (buf == NULL) goto CONSUME_DECODE_OVER;
SDB_GET_BINARY(pRaw, dataPos, buf, len, CONSUME_DECODE_OVER);
tDecodeSMqConsumerObj(buf, pConsumer);
SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CONSUME_DECODE_OVER);
terrno = TSDB_CODE_SUCCESS;
#if 0
SDB_GET_INT32(pRaw, dataPos, &topicNum, CONSUME_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &topicNum, CONSUME_DECODE_OVER);
for (int i = 0; i < topicNum; i++) { for (int i = 0; i < topicNum; i++) {
int32_t topicLen; int32_t topicLen;
...@@ -154,6 +159,7 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { ...@@ -154,6 +159,7 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
int32_t vgSize; int32_t vgSize;
SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER);
} }
#endif
CONSUME_DECODE_OVER: CONSUME_DECODE_OVER:
if (terrno != 0) { if (terrno != 0) {
...@@ -162,8 +168,6 @@ CONSUME_DECODE_OVER: ...@@ -162,8 +168,6 @@ CONSUME_DECODE_OVER:
return NULL; return NULL;
} }
/*SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE);*/
return pRow; return pRow;
} }
...@@ -201,214 +205,13 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) { ...@@ -201,214 +205,13 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) {
sdbRelease(pSdb, pConsumer); sdbRelease(pSdb, pConsumer);
} }
static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { #if 0
SMnode *pMnode = pMsg->pMnode;
char *msgStr = pMsg->rpcMsg.pCont;
SCMSubscribeReq subscribe;
tDeserializeSCMSubscribeReq(msgStr, &subscribe);
int64_t consumerId = subscribe.consumerId;
char *consumerGroup = subscribe.consumerGroup;
int32_t cgroupLen = strlen(consumerGroup);
SArray *newSub = NULL;
int newTopicNum = subscribe.topicNum;
if (newTopicNum) {
newSub = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic));
}
SMqConsumerTopic *pConsumerTopics = calloc(newTopicNum, sizeof(SMqConsumerTopic));
if (pConsumerTopics == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int i = 0; i < newTopicNum; i++) {
char *newTopicName = taosArrayGetP(newSub, i);
SMqConsumerTopic *pConsumerTopic = &pConsumerTopics[i];
strcpy(pConsumerTopic->name, newTopicName);
pConsumerTopic->vgroups = tdListNew(sizeof(int64_t));
}
taosArrayAddBatch(newSub, pConsumerTopics, newTopicNum);
free(pConsumerTopics);
taosArraySortString(newSub, taosArrayCompareString);
SArray *oldSub = NULL;
int oldTopicNum = 0;
// create consumer if not exist
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
// create consumer
pConsumer = malloc(sizeof(SMqConsumerObj));
if (pConsumer == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pConsumer->consumerId = consumerId;
strcpy(pConsumer->cgroup, consumerGroup);
} else {
oldSub = pConsumer->topics;
oldTopicNum = taosArrayGetSize(oldSub);
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);
if (pTrans == NULL) {
//TODO: free memory
return -1;
}
int i = 0, j = 0;
while (i < newTopicNum || j < oldTopicNum) {
SMqConsumerTopic *pOldTopic = NULL;
SMqConsumerTopic *pNewTopic = NULL;
if (i >= newTopicNum) {
// encode unset topic msg to all vnodes related to that topic
pOldTopic = taosArrayGet(oldSub, j);
j++;
} else if (j >= oldTopicNum) {
pNewTopic = taosArrayGet(newSub, i);
i++;
} else {
pNewTopic = taosArrayGet(newSub, i);
pOldTopic = taosArrayGet(oldSub, j);
char *newName = pNewTopic->name;
char *oldName = pOldTopic->name;
int comp = compareLenPrefixedStr(newName, oldName);
if (comp == 0) {
// do nothing
pOldTopic = pNewTopic = NULL;
i++;
j++;
continue;
} else if (comp < 0) {
pOldTopic = NULL;
i++;
} else {
pNewTopic = NULL;
j++;
}
}
if (pOldTopic != NULL) {
//cancel subscribe of that old topic
ASSERT(pNewTopic == NULL);
char *oldTopicName = pOldTopic->name;
SList *vgroups = pOldTopic->vgroups;
SListIter iter;
tdListInitIter(vgroups, &iter, TD_LIST_FORWARD);
SListNode *pn;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, oldTopicName);
ASSERT(pTopic != NULL);
SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen);
while ((pn = tdListNext(&iter)) != NULL) {
int32_t vgId = *(int64_t *)pn->data;
// acquire and get epset
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
// TODO what time to release?
if (pVgObj == NULL) {
// TODO handle error
continue;
}
//build reset msg
void *pMqVgSetReq = mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, consumerId, consumerGroup);
// TODO:serialize
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
action.pCont = pMqVgSetReq;
action.contLen = 0; // TODO
action.msgType = TDMT_VND_MQ_SET_CONN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMqVgSetReq);
mndTransDrop(pTrans);
// TODO free
return -1;
}
}
//delete data in mnode
taosHashRemove(pTopic->cgroups, consumerGroup, cgroupLen);
mndReleaseTopic(pMnode, pTopic);
} else if (pNewTopic != NULL) {
// save subscribe info to mnode
ASSERT(pOldTopic == NULL);
char *newTopicName = pNewTopic->name;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName);
ASSERT(pTopic != NULL);
SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen);
if (pGroup == NULL) {
// add new group
pGroup = malloc(sizeof(SMqCGroup));
if (pGroup == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pGroup->consumerIds = tdListNew(sizeof(int64_t));
if (pGroup->consumerIds == NULL) {
free(pGroup);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pGroup->status = 0;
// add into cgroups
taosHashPut(pTopic->cgroups, consumerGroup, cgroupLen, pGroup, sizeof(SMqCGroup));
}
/*taosHashPut(pTopic->consumers, &pConsumer->consumerId, sizeof(int64_t), pConsumer, sizeof(SMqConsumerObj));*/
// put the consumer into list
// rebalance will be triggered by timer
tdListAppend(pGroup->consumerIds, &consumerId);
SSdbRaw *pTopicRaw = mndTopicActionEncode(pTopic);
sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY);
// TODO: error handling
mndTransAppendRedolog(pTrans, pTopicRaw);
mndReleaseTopic(pMnode, pTopic);
} else {
ASSERT(0);
}
}
// destroy old sub
taosArrayDestroy(oldSub);
// put new sub into consumerobj
pConsumer->topics = newSub;
// persist consumerObj
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
// TODO: error handling
mndTransAppendRedolog(pTrans, pConsumerRaw);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
mndReleaseConsumer(pMnode, pConsumer);
return -1;
}
// TODO: free memory
mndTransDrop(pTrans);
mndReleaseConsumer(pMnode, pConsumer);
return 0;
}
static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
STableInfoReq *pInfo = pMsg->rpcMsg.pCont; STableInfoReq *pInfo = pMsg->rpcMsg.pCont;
mDebug("consumer:%s, start to retrieve meta", pInfo->tableFname); mDebug("consumer:%s, start to retrieve meta", pInfo->tableFname);
#if 0
SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname); SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED; terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
...@@ -463,7 +266,6 @@ static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { ...@@ -463,7 +266,6 @@ static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) {
pMsg->contLen = contLen; pMsg->contLen = contLen;
mDebug("consumer:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pConsumer->numOfColumns, pConsumer->numOfTags); mDebug("consumer:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pConsumer->numOfColumns, pConsumer->numOfTags);
#endif
return 0; return 0;
} }
...@@ -546,3 +348,4 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { ...@@ -546,3 +348,4 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
} }
#endif
source/dnode/mnode/impl/src/mndProfile.c
浏览文件 @ f9d7cc44
...@@ -273,6 +273,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { ...@@ -273,6 +273,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) {
} }
static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) { static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) {
#if 0
SClientHbRsp* pRsp = malloc(sizeof(SClientHbRsp)); SClientHbRsp* pRsp = malloc(sizeof(SClientHbRsp));
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
...@@ -332,6 +333,8 @@ static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) { ...@@ -332,6 +333,8 @@ static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) {
pRsp->body = buf; pRsp->body = buf;
pRsp->bodyLen = tlen; pRsp->bodyLen = tlen;
return pRsp; return pRsp;
#endif
return NULL;
} }
static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
......
source/dnode/mnode/impl/src/mndSubscribe.c 0 → 100644
浏览文件 @ f9d7cc44
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndSubscribe.h"
#include "mndConsumer.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndTopic.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "tcompare.h"
#include "tname.h"
#define MND_SUBSCRIBE_VER_NUMBER 1
#define MND_SUBSCRIBE_RESERVE_SIZE 64
static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *);
static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *);
static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub);
static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg);
static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg);
static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg);
static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg);
static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg);
static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer,
SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic);
int32_t mndInitSubscribe(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
.keyType = SDB_KEY_BINARY,
.encodeFp = (SdbEncodeFp)mndSubActionEncode,
.decodeFp = (SdbDecodeFp)mndSubActionDecode,
.insertFp = (SdbInsertFp)mndSubActionInsert,
.updateFp = (SdbUpdateFp)mndSubActionUpdate,
.deleteFp = (SdbDeleteFp)mndSubActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
/*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/
/*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/
mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
return sdbSetTable(pMnode->pSdb, table);
}
static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) {
int i = 0;
while (key[i] != ':') {
i++;
}
key[i] = 0;
*topic = strdup(key);
key[i] = ':';
*cgroup = strdup(&key[i + 1]);
return 0;
}
static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SSdb *pSdb = pMnode->pSdb;
SMqSubscribeObj *pSub = NULL;
void *pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub);
int sz;
while (pIter != NULL) {
if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) {
char *topic = NULL;
char *cgroup = NULL;
mndSplitSubscribeKey(pSub->key, &topic, &cgroup);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
// create trans
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);
for (int i = 0; i < sz; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx);
SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg);
pCEp->consumerId = consumerId;
taosArrayPush(pSub->assigned, pCEp);
pSub->nextConsumerIdx++;
// build msg
SMqSetCVgReq req = {
.vgId = pCEp->vgId,
.consumerId = consumerId,
};
strcpy(req.cGroup, cgroup);
strcpy(req.topicName, topic);
strcpy(req.sql, pTopic->sql);
strcpy(req.logicalPlan, pTopic->logicalPlan);
strcpy(req.physicalPlan, pTopic->physicalPlan);
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
void *reqStr = malloc(tlen);
if (reqStr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
void *abuf = reqStr;
tEncodeSMqSetCVgReq(abuf, &req);
// persist msg
STransAction action = {0};
action.epSet = pCEp->epset;
action.pCont = reqStr;
action.contLen = tlen;
action.msgType = TDMT_VND_MQ_SET_CONN;
mndTransAppendRedoAction(pTrans, &action);
// persist raw
SSdbRaw *pRaw = mndSubActionEncode(pSub);
mndTransAppendRedolog(pTrans, pRaw);
tfree(topic);
tfree(cgroup);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
}
mndReleaseTopic(pMnode, pTopic);
mndTransDrop(pTrans);
}
pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub);
}
return 0;
}
static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) {
SMqConsumerEp CEp;
CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1;
int32_t sz;
SVgObj *pVgroup = NULL;
SSdb *pSdb = pMnode->pSdb;
void *pIter = sdbFetch(pSdb, SDB_VGROUP, NULL, (void **)&pVgroup);
while (pIter != NULL) {
if (pVgroup->dbUid == pTopic->dbUid) {
CEp.epset = mndGetVgroupEpset(pMnode, pVgroup);
CEp.vgId = pVgroup->vgId;
taosArrayPush(unassignedVg, &CEp);
}
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
}
return 0;
}
static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer,
SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic) {
int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo);
for (int32_t i = 0; i < sz; i++) {
int32_t vgId = *(int32_t *)taosArrayGet(pConsumerTopic->pVgInfo, i);
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
SMqSetCVgReq req = {
.vgId = vgId,
.consumerId = pConsumer->consumerId,
};
strcpy(req.cGroup, pConsumer->cgroup);
strcpy(req.topicName, pTopic->name);
strcpy(req.sql, pTopic->sql);
strcpy(req.logicalPlan, pTopic->logicalPlan);
strcpy(req.physicalPlan, pTopic->physicalPlan);
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
void *reqStr = malloc(tlen);
if (reqStr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
void *abuf = reqStr;
tEncodeSMqSetCVgReq(&abuf, &req);
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
action.pCont = reqStr;
action.contLen = tlen;
action.msgType = TDMT_VND_MQ_SET_CONN;
mndReleaseVgroup(pMnode, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(reqStr);
return -1;
}
}
return 0;
}
void mndCleanupSubscribe(SMnode *pMnode) {}
static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) {
int32_t tlen = tEncodeSubscribeObj(NULL, pSub);
int32_t size = tlen + MND_SUBSCRIBE_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size);
if (pRaw == NULL) goto SUB_ENCODE_OVER;
void *buf = malloc(tlen);
if (buf == NULL) {
goto SUB_ENCODE_OVER;
}
void *abuf = buf;
tEncodeSubscribeObj(&buf, pSub);
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, SUB_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, SUB_ENCODE_OVER);
SDB_SET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, SUB_ENCODE_OVER);
SUB_ENCODE_OVER:
if (terrno != 0) {
mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("subscribe:%s, encode to raw:%p, row:%p", pSub->key, pRaw, pSub);
return pRaw;
}
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER;
if (sver != MND_SUBSCRIBE_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto SUB_DECODE_OVER;
}
int32_t size = sizeof(SMqSubscribeObj);
SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto SUB_DECODE_OVER;
SMqSubscribeObj *pSub = sdbGetRowObj(pRow);
if (pSub == NULL) goto SUB_DECODE_OVER;
int32_t dataPos = 0;
int32_t tlen;
void *buf = malloc(tlen + 1);
if (buf == NULL) goto SUB_DECODE_OVER;
SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER);
if (tDecodeSubscribeObj(buf, pSub) == NULL) {
goto SUB_DECODE_OVER;
}
SUB_DECODE_OVER:
if (terrno != 0) {
mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr());
// TODO free subscribeobj
tfree(pRow);
return NULL;
}
return pRow;
}
static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) {
mTrace("subscribe:%s, perform insert action", pSub->key);
return 0;
}
static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) {
mTrace("subscribe:%s, perform delete action", pSub->key);
return 0;
}
static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub) {
mTrace("subscribe:%s, perform update action", pOldSub->key);
return 0;
}
static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) {
return 0;
}
static char *mndMakeSubscribeKey(char *cgroup, char *topicName) {
char *key = malloc(TSDB_SHOW_SUBQUERY_LEN);
if (key == NULL) {
return NULL;
}
int tlen = strlen(cgroup);
memcpy(key, cgroup, tlen);
key[tlen] = ':';
strcpy(key + tlen + 1, topicName);
return key;
}
SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, char *cgroup, char *topicName) {
SSdb *pSdb = pMnode->pSdb;
char *key = mndMakeSubscribeKey(cgroup, topicName);
SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key);
free(key);
if (pSub == NULL) {
/*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/
}
return pSub;
}
void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pSub);
}
static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
char *msgStr = pMsg->rpcMsg.pCont;
SCMSubscribeReq subscribe;
tDeserializeSCMSubscribeReq(msgStr, &subscribe);
int64_t consumerId = subscribe.consumerId;
char *consumerGroup = subscribe.consumerGroup;
int32_t cgroupLen = strlen(consumerGroup);
SArray *newSub = subscribe.topicNames;
int newTopicNum = subscribe.topicNum;
taosArraySortString(newSub, taosArrayCompareString);
SArray *oldSub = NULL;
int oldTopicNum = 0;
// create consumer if not exist
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
// create consumer
pConsumer = malloc(sizeof(SMqConsumerObj));
if (pConsumer == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pConsumer->consumerId = consumerId;
strcpy(pConsumer->cgroup, consumerGroup);
taosInitRWLatch(&pConsumer->lock);
} else {
oldSub = pConsumer->topics;
}
pConsumer->topics = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic));
if (oldSub != NULL) {
oldTopicNum = taosArrayGetSize(oldSub);
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);
if (pTrans == NULL) {
// TODO: free memory
return -1;
}
int i = 0, j = 0;
while (i < newTopicNum || j < oldTopicNum) {
char *newTopicName = NULL;
char *oldTopicName = NULL;
if (i >= newTopicNum) {
// encode unset topic msg to all vnodes related to that topic
oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name;
j++;
} else if (j >= oldTopicNum) {
newTopicName = taosArrayGet(newSub, i);
i++;
} else {
newTopicName = taosArrayGet(newSub, i);
oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name;
int comp = compareLenPrefixedStr(newTopicName, oldTopicName);
if (comp == 0) {
// do nothing
oldTopicName = newTopicName = NULL;
i++;
j++;
continue;
} else if (comp < 0) {
oldTopicName = NULL;
i++;
} else {
newTopicName = NULL;
j++;
}
}
if (oldTopicName != NULL) {
#if 0
// cancel subscribe of that old topic
ASSERT(pNewTopic == NULL);
char *oldTopicName = pOldTopic->name;
SList *vgroups = pOldTopic->vgroups;
SListIter iter;
tdListInitIter(vgroups, &iter, TD_LIST_FORWARD);
SListNode *pn;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, oldTopicName);
ASSERT(pTopic != NULL);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, oldTopicName);
SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen);
while ((pn = tdListNext(&iter)) != NULL) {
int32_t vgId = *(int64_t *)pn->data;
// acquire and get epset
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
// TODO what time to release?
if (pVgObj == NULL) {
// TODO handle error
continue;
}
// build reset msg
void *pMqVgSetReq = mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, consumerId, consumerGroup);
// TODO:serialize
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
action.pCont = pMqVgSetReq;
action.contLen = 0; // TODO
action.msgType = TDMT_VND_MQ_SET_CONN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMqVgSetReq);
mndTransDrop(pTrans);
// TODO free
return -1;
}
}
// delete data in mnode
taosHashRemove(pTopic->cgroups, consumerGroup, cgroupLen);
mndReleaseSubscribe(pMnode, pSub);
mndReleaseTopic(pMnode, pTopic);
#endif
} else if (newTopicName != NULL) {
// save subscribe info to mnode
ASSERT(oldTopicName == NULL);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName);
if (pTopic == NULL) {
/*terrno = */
continue;
}
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName);
if (pSub == NULL) {
pSub = tNewSubscribeObj();
if (pSub == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
// set unassigned vg
mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg);
}
taosArrayPush(pSub->availConsumer, &consumerId);
//TODO: no need
SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub);
taosArrayPush(pConsumer->topics, pConsumerTopic);
if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) {
int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo);
// send setmsg to vnode
if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic) < 0) {
// TODO
return -1;
}
}
taosArrayDestroy(pConsumerTopic->pVgInfo);
free(pConsumerTopic);
SSdbRaw *pRaw = mndSubActionEncode(pSub);
/*sdbSetRawStatus(pRaw, SDB_STATUS_READY);*/
mndTransAppendRedolog(pTrans, pRaw);
#if 0
SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen);
if (pGroup == NULL) {
// add new group
pGroup = malloc(sizeof(SMqCGroup));
if (pGroup == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pGroup->consumerIds = tdListNew(sizeof(int64_t));
if (pGroup->consumerIds == NULL) {
free(pGroup);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pGroup->status = 0;
// add into cgroups
taosHashPut(pTopic->cgroups, consumerGroup, cgroupLen, pGroup, sizeof(SMqCGroup));
}
/*taosHashPut(pTopic->consumers, &pConsumer->consumerId, sizeof(int64_t), pConsumer, sizeof(SMqConsumerObj));*/
// put the consumer into list
// rebalance will be triggered by timer
tdListAppend(pGroup->consumerIds, &consumerId);
SSdbRaw *pTopicRaw = mndTopicActionEncode(pTopic);
sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY);
// TODO: error handling
mndTransAppendRedolog(pTrans, pTopicRaw);
#endif
mndReleaseTopic(pMnode, pTopic);
mndReleaseSubscribe(pMnode, pSub);
}
}
// part3. persist consumerObj
// destroy old sub
if (oldSub) taosArrayDestroy(oldSub);
// put new sub into consumerobj
// persist consumerObj
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
// TODO: error handling
mndTransAppendRedolog(pTrans, pConsumerRaw);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
mndReleaseConsumer(pMnode, pConsumer);
return -1;
}
// TODO: free memory
if (newSub) taosArrayDestroy(newSub);
mndTransDrop(pTrans);
mndReleaseConsumer(pMnode, pConsumer);
return 0;
}
static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
STableInfoReq *pInfo = pMsg->rpcMsg.pCont;
mDebug("subscribe:%s, start to retrieve meta", pInfo->tableFname);
#if 0
SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
mError("consumer:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr());
return -1;
}
SConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pInfo->tableFname);
if (pConsumer == NULL) {
mndReleaseDb(pMnode, pDb);
terrno = TSDB_CODE_MND_INVALID_CONSUMER;
mError("consumer:%s, failed to get meta since %s", pInfo->tableFname, terrstr());
return -1;
}
taosRLockLatch(&pConsumer->lock);
int32_t totalCols = pConsumer->numOfColumns + pConsumer->numOfTags;
int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema);
STableMetaRsp *pMeta = rpcMallocCont(contLen);
if (pMeta == NULL) {
taosRUnLockLatch(&pConsumer->lock);
mndReleaseDb(pMnode, pDb);
mndReleaseConsumer(pMnode, pConsumer);
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("consumer:%s, failed to get meta since %s", pInfo->tableFname, terrstr());
return -1;
}
memcpy(pMeta->consumerFname, pConsumer->name, TSDB_TABLE_FNAME_LEN);
pMeta->numOfTags = htonl(pConsumer->numOfTags);
pMeta->numOfColumns = htonl(pConsumer->numOfColumns);
pMeta->precision = pDb->cfg.precision;
pMeta->tableType = TSDB_SUPER_TABLE;
pMeta->update = pDb->cfg.update;
pMeta->sversion = htonl(pConsumer->version);
pMeta->tuid = htonl(pConsumer->uid);
for (int32_t i = 0; i < totalCols; ++i) {
SSchema *pSchema = &pMeta->pSchema[i];
SSchema *pSrcSchema = &pConsumer->pSchema[i];
memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
pSchema->type = pSrcSchema->type;
pSchema->colId = htonl(pSrcSchema->colId);
pSchema->bytes = htonl(pSrcSchema->bytes);
}
taosRUnLockLatch(&pConsumer->lock);
mndReleaseDb(pMnode, pDb);
mndReleaseConsumer(pMnode, pConsumer);
pMsg->pCont = pMeta;
pMsg->contLen = contLen;
mDebug("consumer:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pConsumer->numOfColumns, pConsumer->numOfTags);
#endif
return 0;
}
static int32_t mndGetNumOfConsumers(SMnode *pMnode, char *dbName, int32_t *pNumOfConsumers) {
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
}
int32_t numOfConsumers = 0;
void *pIter = NULL;
while (1) {
SMqConsumerObj *pConsumer = NULL;
pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) break;
numOfConsumers++;
sdbRelease(pSdb, pConsumer);
}
*pNumOfConsumers = numOfConsumers;
return 0;
}
static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) {
SMnode *pMnode = pMsg->pMnode;
SSdb *pSdb = pMnode->pSdb;
if (mndGetNumOfConsumers(pMnode, pShow->db, &pShow->numOfRows) != 0) {
return -1;
}
int32_t cols = 0;
SSchema *pSchema = pMeta->pSchema;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "columns");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "tags");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = sdbGetSize(pSdb, SDB_CONSUMER);
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbFname, mndShowStr(pShow->type));
return 0;
}
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}
source/dnode/mnode/impl/src/mndTopic.c
浏览文件 @ f9d7cc44
...@@ -79,8 +79,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { ...@@ -79,8 +79,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, logicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, logicalPlanLen, TOPIC_ENCODE_OVER);
int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
pTopic->physicalPlan = calloc(physicalPlanLen, sizeof(char));
if (pTopic->physicalPlan == NULL) goto TOPIC_ENCODE_OVER;
SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->physicalPlan)+1, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->physicalPlan)+1, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER);
...@@ -92,12 +90,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { ...@@ -92,12 +90,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
TOPIC_ENCODE_OVER: TOPIC_ENCODE_OVER:
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
mError("topic:%s, failed to encode to raw:%p since %s", pTopic->name, pRaw, terrstr()); mError("topic:%s, failed to encode to raw:%p since %s", pTopic->name, pRaw, terrstr());
/*if (pTopic->logicalPlan) {*/
/*free(pTopic->logicalPlan);*/
/*}*/
/*if (pTopic->physicalPlan) {*/
/*free(pTopic->physicalPlan);*/
/*}*/
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
return NULL; return NULL;
} }
...@@ -138,7 +130,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { ...@@ -138,7 +130,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
pTopic->logicalPlan = calloc(len+1, sizeof(char)); pTopic->logicalPlan = calloc(len + 1, sizeof(char));
if (pTopic->logicalPlan == NULL) { if (pTopic->logicalPlan == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto TOPIC_DECODE_OVER; goto TOPIC_DECODE_OVER;
...@@ -146,7 +138,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { ...@@ -146,7 +138,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len+1, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len+1, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
pTopic->logicalPlan = calloc(len + 1, sizeof(char)); pTopic->physicalPlan = calloc(len + 1, sizeof(char));
if (pTopic->physicalPlan == NULL) { if (pTopic->physicalPlan == NULL) {
free(pTopic->logicalPlan); free(pTopic->logicalPlan);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
...@@ -154,7 +146,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { ...@@ -154,7 +146,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
} }
SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len+1, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len+1, TOPIC_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER) SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER);
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
......
source/dnode/mnode/impl/src/mnode.c
浏览文件 @ f9d7cc44
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "mndAuth.h" #include "mndAuth.h"
#include "mndBnode.h" #include "mndBnode.h"
#include "mndCluster.h" #include "mndCluster.h"
#include "mndConsumer.h"
#include "mndDb.h" #include "mndDb.h"
#include "mndDnode.h" #include "mndDnode.h"
#include "mndFunc.h" #include "mndFunc.h"
...@@ -27,6 +28,7 @@ ...@@ -27,6 +28,7 @@
#include "mndShow.h" #include "mndShow.h"
#include "mndSnode.h" #include "mndSnode.h"
#include "mndStb.h" #include "mndStb.h"
#include "mndSubscribe.h"
#include "mndSync.h" #include "mndSync.h"
#include "mndTelem.h" #include "mndTelem.h"
#include "mndTopic.h" #include "mndTopic.h"
...@@ -69,15 +71,15 @@ static void mndTransReExecute(void *param, void *tmrId) { ...@@ -69,15 +71,15 @@ static void mndTransReExecute(void *param, void *tmrId) {
taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer); taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer);
} }
static void mndCalMqRebalance(void* param, void* tmrId) { static void mndCalMqRebalance(void *param, void *tmrId) {
SMnode* pMnode = param; SMnode *pMnode = param;
if (mndIsMaster(pMnode)) { if (mndIsMaster(pMnode)) {
// iterate cgroup, cal rebalance SMqTmrMsg *pMsg = rpcMallocCont(sizeof(SMqTmrMsg));
// sync with raft SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pMsg, .contLen = sizeof(SMqTmrMsg)};
// write sdb pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);
} }
taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->transTimer); taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer);
} }
static int32_t mndInitTimer(SMnode *pMnode) { static int32_t mndInitTimer(SMnode *pMnode) {
...@@ -95,6 +97,11 @@ static int32_t mndInitTimer(SMnode *pMnode) { ...@@ -95,6 +97,11 @@ static int32_t mndInitTimer(SMnode *pMnode) {
return -1; return -1;
} }
if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0; return 0;
} }
...@@ -102,6 +109,8 @@ static void mndCleanupTimer(SMnode *pMnode) { ...@@ -102,6 +109,8 @@ static void mndCleanupTimer(SMnode *pMnode) {
if (pMnode->timer != NULL) { if (pMnode->timer != NULL) {
taosTmrStop(pMnode->transTimer); taosTmrStop(pMnode->transTimer);
pMnode->transTimer = NULL; pMnode->transTimer = NULL;
taosTmrStop(pMnode->mqTimer);
pMnode->mqTimer = NULL;
taosTmrCleanUp(pMnode->timer); taosTmrCleanUp(pMnode->timer);
pMnode->timer = NULL; pMnode->timer = NULL;
} }
...@@ -171,6 +180,8 @@ static int32_t mndInitSteps(SMnode *pMnode) { ...@@ -171,6 +180,8 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
...@@ -377,7 +388,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { ...@@ -377,7 +388,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
return NULL; return NULL;
} }
if (pRpcMsg->msgType != TDMT_MND_TRANS) { if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER) {
SRpcConnInfo connInfo = {0}; SRpcConnInfo connInfo = {0};
if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) {
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
......
source/dnode/mnode/sdb/src/sdbHash.c
浏览文件 @ f9d7cc44
...@@ -38,10 +38,10 @@ const char *sdbTableName(ESdbType type) { ...@@ -38,10 +38,10 @@ const char *sdbTableName(ESdbType type) {
return "auth"; return "auth";
case SDB_ACCT: case SDB_ACCT:
return "acct"; return "acct";
case SDB_SUBSCRIBE:
return "subscribe";
case SDB_CONSUMER: case SDB_CONSUMER:
return "consumer"; return "consumer";
case SDB_CGROUP:
return "cgroup";
case SDB_TOPIC: case SDB_TOPIC:
return "topic"; return "topic";
case SDB_VGROUP: case SDB_VGROUP:
......
source/dnode/vnode/inc/tsdb.h
浏览文件 @ f9d7cc44
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
#include "mallocator.h" #include "mallocator.h"
#include "meta.h" #include "meta.h"
#include "common.h" #include "common.h"
#include "tfs.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
...@@ -80,7 +81,7 @@ typedef struct { ...@@ -80,7 +81,7 @@ typedef struct {
} STableKeyInfo; } STableKeyInfo;
// STsdb // STsdb
STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta); STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs);
void tsdbClose(STsdb *); void tsdbClose(STsdb *);
void tsdbRemove(const char *path); void tsdbRemove(const char *path);
int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp); int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp);
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ f9d7cc44
...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
#include "meta.h" #include "meta.h"
#include "tarray.h" #include "tarray.h"
#include "tfs.h"
#include "tq.h" #include "tq.h"
#include "tsdb.h" #include "tsdb.h"
#include "wal.h" #include "wal.h"
...@@ -36,7 +37,8 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); ...@@ -36,7 +37,8 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq);
typedef struct SVnodeCfg { typedef struct SVnodeCfg {
int32_t vgId; int32_t vgId;
SDnode * pDnode; SDnode *pDnode;
STfs *pTfs;
uint64_t wsize; uint64_t wsize;
uint64_t ssize; uint64_t ssize;
uint64_t lsize; uint64_t lsize;
...@@ -52,9 +54,9 @@ typedef struct SVnodeCfg { ...@@ -52,9 +54,9 @@ typedef struct SVnodeCfg {
typedef struct { typedef struct {
int32_t sver; int32_t sver;
char * timezone; const char *timezone;
char * locale; const char *locale;
char * charset; const char *charset;
uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO)
PutReqToVQueryQFp putReqToVQueryQFp; PutReqToVQueryQFp putReqToVQueryQFp;
} SVnodeOpt; } SVnodeOpt;
......
source/dnode/vnode/src/inc/tsdbDef.h
浏览文件 @ f9d7cc44
...@@ -50,6 +50,7 @@ struct STsdb { ...@@ -50,6 +50,7 @@ struct STsdb {
SMemAllocatorFactory *pmaf; SMemAllocatorFactory *pmaf;
STsdbFS * fs; STsdbFS * fs;
SMeta * pMeta; SMeta * pMeta;
STfs * pTfs;
}; };
#define REPO_ID(r) ((r)->vgId) #define REPO_ID(r) ((r)->vgId)
......
source/dnode/vnode/src/inc/tsdbFile.h
浏览文件 @ f9d7cc44
...@@ -29,12 +29,15 @@ ...@@ -29,12 +29,15 @@
#define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_INFO(tf) (&((tf)->info))
#define TSDB_FILE_F(tf) (&((tf)->f)) #define TSDB_FILE_F(tf) (&((tf)->f))
#define TSDB_FILE_FD(tf) ((tf)->fd) #define TSDB_FILE_FD(tf) ((tf)->fd)
#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf)) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) #define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) #define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf)) #define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf)) #define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf)) #define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf))
#define TSDB_FILE_STATE(tf) ((tf)->state) #define TSDB_FILE_STATE(tf) ((tf)->state)
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
...@@ -54,10 +57,10 @@ typedef struct { ...@@ -54,10 +57,10 @@ typedef struct {
} SMFInfo; } SMFInfo;
typedef struct { typedef struct {
SMFInfo info; SMFInfo info;
TFILE f; STfsFile f;
int fd; int fd;
uint8_t state; uint8_t state;
} SMFile; } SMFile;
void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
...@@ -175,17 +178,17 @@ typedef struct { ...@@ -175,17 +178,17 @@ typedef struct {
} SDFInfo; } SDFInfo;
typedef struct { typedef struct {
SDFInfo info; SDFInfo info;
TFILE f; STfsFile f;
int fd; int fd;
uint8_t state; uint8_t state;
} SDFile; } SDFile;
void tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype); void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
int tsdbEncodeSDFile(void** buf, SDFile* pDFile); int tsdbEncodeSDFile(void** buf, SDFile* pDFile);
void* tsdbDecodeSDFile(void* buf, SDFile* pDFile); void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile);
int tsdbCreateDFile(SDFile* pDFile, bool updateHeader); int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader);
int tsdbUpdateDFileHeader(SDFile* pDFile); int tsdbUpdateDFileHeader(SDFile* pDFile);
int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version); int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
...@@ -263,7 +266,7 @@ static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte ...@@ -263,7 +266,7 @@ static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte
return (int)nbyte; return (int)nbyte;
} }
static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); } static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); }
static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile)); ASSERT(TSDB_FILE_OPENED(pDFile));
...@@ -278,7 +281,7 @@ static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nby ...@@ -278,7 +281,7 @@ static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nby
} }
static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
...@@ -311,14 +314,14 @@ typedef struct { ...@@ -311,14 +314,14 @@ typedef struct {
} \ } \
} while (0); } while (0);
void tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver); void tsdbInitDFileSet(STsdb *pRepo, SDFileSet* pSet, SDiskID did, int fid, uint32_t ver);
void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet); void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet); int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet); void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet);
int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet); int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet); void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to); int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
int tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader); int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet* pSet, bool updateHeader);
int tsdbUpdateDFileSetHeader(SDFileSet* pSet); int tsdbUpdateDFileSetHeader(SDFileSet* pSet);
int tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet); int tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet);
......
source/dnode/vnode/src/inc/vnd.h
浏览文件 @ f9d7cc44
...@@ -78,6 +78,7 @@ struct SVnode { ...@@ -78,6 +78,7 @@ struct SVnode {
tsem_t canCommit; tsem_t canCommit;
SQHandle* pQuery; SQHandle* pQuery;
SDnode* pDnode; SDnode* pDnode;
STfs* pTfs;
}; };
int vnodeScheduleTask(SVnodeTask* task); int vnodeScheduleTask(SVnodeTask* task);
......
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ f9d7cc44
...@@ -97,15 +97,14 @@ int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { ...@@ -97,15 +97,14 @@ int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
level = tsdbGetFidLevel(pSet->fid, pRtn); level = tsdbGetFidLevel(pSet->fid, pRtn);
tfsAllocDisk(level, &(did.level), &(did.id)); if (tfsAllocDisk(pRepo->pTfs, level, &did) < 0) {
if (did.level == TFS_UNDECIDED_LEVEL) {
terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
return -1; return -1;
} }
if (did.level > TSDB_FSET_LEVEL(pSet)) { if (did.level > TSDB_FSET_LEVEL(pSet)) {
// Need to move the FSET to higher level // Need to move the FSET to higher level
tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); tsdbInitDFileSet(pRepo, &nSet, did, pSet->fid, FS_TXN_VERSION(pfs));
if (tsdbCopyDFileSet(pSet, &nSet) < 0) { if (tsdbCopyDFileSet(pSet, &nSet) < 0) {
tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid,
...@@ -456,8 +455,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid ...@@ -456,8 +455,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); STsdb * pRepo = TSDB_COMMIT_REPO(pCommith);
SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith);
tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id)); if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) {
if (did.level == TFS_UNDECIDED_LEVEL) {
terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
return -1; return -1;
} }
...@@ -484,9 +482,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid ...@@ -484,9 +482,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
// Set and open commit FSET // Set and open commit FSET
if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) {
// Create a new FSET to write data // Create a new FSET to write data
tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); tsdbInitDFileSet(pRepo, pWSet, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)));
if (tsdbCreateDFileSet(pWSet, true) < 0) { if (tsdbCreateDFileSet(pRepo, pWSet, true) < 0) {
tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo),
TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno));
if (pCommith->isRFileSet) { if (pCommith->isRFileSet) {
...@@ -509,8 +507,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid ...@@ -509,8 +507,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
// TSDB_FILE_HEAD // TSDB_FILE_HEAD
SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith);
tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD);
if (tsdbCreateDFile(pWHeadf, true) < 0) { if (tsdbCreateDFile(pRepo, pWHeadf, true) < 0) {
tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf),
tstrerror(terrno)); tstrerror(terrno));
...@@ -556,10 +554,10 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid ...@@ -556,10 +554,10 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
} }
} }
} else { } else {
tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); tsdbInitDFile(pRepo, pWLastf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST);
pCommith->isLFileSame = false; pCommith->isLFileSame = false;
if (tsdbCreateDFile(pWLastf, true) < 0) { if (tsdbCreateDFile(pRepo, pWLastf, true) < 0) {
tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf),
tstrerror(terrno)); tstrerror(terrno));
......
source/dnode/vnode/src/tsdb/tsdbCompact.c
浏览文件 @ f9d7cc44
...@@ -186,8 +186,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { ...@@ -186,8 +186,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) {
} }
} else { } else {
// Create new fset as compacted fset // Create new fset as compacted fset
tfsAllocDisk(tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &(did.level), &(did.id)); if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &did) < 0) {
if (did.level == TFS_UNDECIDED_LEVEL) {
terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno)); tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
tsdbCompactFSetEnd(pComph); tsdbCompactFSetEnd(pComph);
......
source/dnode/vnode/src/tsdb/tsdbFS.c
浏览文件 @ f9d7cc44
...@@ -23,14 +23,14 @@ static const char *tsdbTxnFname[] = {"current.t", "current"}; ...@@ -23,14 +23,14 @@ static const char *tsdbTxnFname[] = {"current.t", "current"};
static int tsdbComparFidFSet(const void *arg1, const void *arg2); static int tsdbComparFidFSet(const void *arg1, const void *arg2);
static void tsdbResetFSStatus(SFSStatus *pStatus); static void tsdbResetFSStatus(SFSStatus *pStatus);
static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid); static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus);
static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo); static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo);
static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]); static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]);
static int tsdbOpenFSFromCurrent(STsdb *pRepo); static int tsdbOpenFSFromCurrent(STsdb *pRepo);
static int tsdbScanAndTryFixFS(STsdb *pRepo); static int tsdbScanAndTryFixFS(STsdb *pRepo);
static int tsdbScanRootDir(STsdb *pRepo); static int tsdbScanRootDir(STsdb *pRepo);
static int tsdbScanDataDir(STsdb *pRepo); static int tsdbScanDataDir(STsdb *pRepo);
static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf); static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf);
static int tsdbRestoreCurrent(STsdb *pRepo); static int tsdbRestoreCurrent(STsdb *pRepo);
static int tsdbComparTFILE(const void *arg1, const void *arg2); static int tsdbComparTFILE(const void *arg1, const void *arg2);
static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired);
...@@ -97,7 +97,7 @@ static int tsdbEncodeDFileSetArray(void **buf, SArray *pArray) { ...@@ -97,7 +97,7 @@ static int tsdbEncodeDFileSetArray(void **buf, SArray *pArray) {
return tlen; return tlen;
} }
static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) { static void *tsdbDecodeDFileSetArray(STsdb*pRepo, void *buf, SArray *pArray) {
uint64_t nset; uint64_t nset;
SDFileSet dset; SDFileSet dset;
...@@ -105,7 +105,7 @@ static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) { ...@@ -105,7 +105,7 @@ static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) {
buf = taosDecodeFixedU64(buf, &nset); buf = taosDecodeFixedU64(buf, &nset);
for (size_t i = 0; i < nset; i++) { for (size_t i = 0; i < nset; i++) {
buf = tsdbDecodeDFileSet(buf, &dset); buf = tsdbDecodeDFileSet(pRepo, buf, &dset);
taosArrayPush(pArray, (void *)(&dset)); taosArrayPush(pArray, (void *)(&dset));
} }
return buf; return buf;
...@@ -122,13 +122,13 @@ static int tsdbEncodeFSStatus(void **buf, SFSStatus *pStatus) { ...@@ -122,13 +122,13 @@ static int tsdbEncodeFSStatus(void **buf, SFSStatus *pStatus) {
return tlen; return tlen;
} }
static void *tsdbDecodeFSStatus(void *buf, SFSStatus *pStatus) { static void *tsdbDecodeFSStatus(STsdb*pRepo, void *buf, SFSStatus *pStatus) {
tsdbResetFSStatus(pStatus); tsdbResetFSStatus(pStatus);
// pStatus->pmf = &(pStatus->mf); // pStatus->pmf = &(pStatus->mf);
// buf = tsdbDecodeSMFile(buf, pStatus->pmf); // buf = tsdbDecodeSMFile(buf, pStatus->pmf);
buf = tsdbDecodeDFileSetArray(buf, pStatus->df); buf = tsdbDecodeDFileSetArray(pRepo, buf, pStatus->df);
return buf; return buf;
} }
...@@ -311,7 +311,7 @@ int tsdbOpenFS(STsdb *pRepo) { ...@@ -311,7 +311,7 @@ int tsdbOpenFS(STsdb *pRepo) {
ASSERT(pfs != NULL); ASSERT(pfs != NULL);
tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
tsdbGetRtnSnap(pRepo, &pRepo->rtn); tsdbGetRtnSnap(pRepo, &pRepo->rtn);
if (access(current, F_OK) == 0) { if (access(current, F_OK) == 0) {
...@@ -375,7 +375,7 @@ int tsdbEndFSTxn(STsdb *pRepo) { ...@@ -375,7 +375,7 @@ int tsdbEndFSTxn(STsdb *pRepo) {
SFSStatus *pStatus; SFSStatus *pStatus;
// Write current file system snapshot // Write current file system snapshot
if (tsdbSaveFSStatus(pfs->nstatus, REPO_ID(pRepo)) < 0) { if (tsdbSaveFSStatus(pRepo, pfs->nstatus) < 0) {
tsdbEndFSTxnWithError(pfs); tsdbEndFSTxnWithError(pfs);
return -1; return -1;
} }
...@@ -405,7 +405,7 @@ int tsdbEndFSTxnWithError(STsdbFS *pfs) { ...@@ -405,7 +405,7 @@ int tsdbEndFSTxnWithError(STsdbFS *pfs) {
int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); } int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); }
static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) { static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
SFSHeader fsheader; SFSHeader fsheader;
void * pBuf = NULL; void * pBuf = NULL;
void * ptr; void * ptr;
...@@ -413,8 +413,8 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) { ...@@ -413,8 +413,8 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
char tfname[TSDB_FILENAME_LEN] = "\0"; char tfname[TSDB_FILENAME_LEN] = "\0";
char cfname[TSDB_FILENAME_LEN] = "\0"; char cfname[TSDB_FILENAME_LEN] = "\0";
tsdbGetTxnFname(vid, TSDB_TXN_TEMP_FILE, tfname); tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname);
tsdbGetTxnFname(vid, TSDB_TXN_CURR_FILE, cfname); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname);
int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (fd < 0) { if (fd < 0) {
...@@ -645,8 +645,9 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { ...@@ -645,8 +645,9 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
} }
} }
static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]) { static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", TFS_PRIMARY_PATH(), repoid, tsdbTxnFname[ftype]); snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(pRepo->pTfs), pRepo->vgId,
tsdbTxnFname[ftype]);
} }
static int tsdbOpenFSFromCurrent(STsdb *pRepo) { static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
...@@ -657,7 +658,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { ...@@ -657,7 +658,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
char current[TSDB_FILENAME_LEN] = "\0"; char current[TSDB_FILENAME_LEN] = "\0";
void * ptr; void * ptr;
tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
// current file exists, try to recover // current file exists, try to recover
fd = open(current, O_RDONLY | O_BINARY); fd = open(current, O_RDONLY | O_BINARY);
...@@ -725,7 +726,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { ...@@ -725,7 +726,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
} }
ptr = buffer; ptr = buffer;
ptr = tsdbDecodeFSStatus(ptr, pStatus); ptr = tsdbDecodeFSStatus(pRepo, ptr, pStatus);
} else { } else {
tsdbResetFSStatus(pStatus); tsdbResetFSStatus(pStatus);
} }
...@@ -910,17 +911,17 @@ static int tsdbScanRootDir(STsdb *pRepo) { ...@@ -910,17 +911,17 @@ static int tsdbScanRootDir(STsdb *pRepo) {
char rootDir[TSDB_FILENAME_LEN]; char rootDir[TSDB_FILENAME_LEN];
char bname[TSDB_FILENAME_LEN]; char bname[TSDB_FILENAME_LEN];
STsdbFS * pfs = REPO_FS(pRepo); STsdbFS * pfs = REPO_FS(pRepo);
const TFILE *pf; const STfsFile *pf;
tsdbGetRootDir(REPO_ID(pRepo), rootDir); tsdbGetRootDir(REPO_ID(pRepo), rootDir);
TDIR *tdir = tfsOpendir(rootDir); STfsDir *tdir = tfsOpendir(pRepo->pTfs, rootDir);
if (tdir == NULL) { if (tdir == NULL) {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
return -1; return -1;
} }
while ((pf = tfsReaddir(tdir))) { while ((pf = tfsReaddir(tdir))) {
tfsbasename(pf, bname); tfsBasename(pf, bname);
if (strcmp(bname, tsdbTxnFname[TSDB_TXN_CURR_FILE]) == 0 || strcmp(bname, "data") == 0) { if (strcmp(bname, tsdbTxnFname[TSDB_TXN_CURR_FILE]) == 0 || strcmp(bname, "data") == 0) {
// Skip current file and data directory // Skip current file and data directory
...@@ -931,8 +932,8 @@ static int tsdbScanRootDir(STsdb *pRepo) { ...@@ -931,8 +932,8 @@ static int tsdbScanRootDir(STsdb *pRepo) {
// continue; // continue;
// } // }
(void)tfsremove(pf); (void)tfsRemoveFile(pf);
tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), pf->aname);
} }
tfsClosedir(tdir); tfsClosedir(tdir);
...@@ -944,21 +945,21 @@ static int tsdbScanDataDir(STsdb *pRepo) { ...@@ -944,21 +945,21 @@ static int tsdbScanDataDir(STsdb *pRepo) {
char dataDir[TSDB_FILENAME_LEN]; char dataDir[TSDB_FILENAME_LEN];
char bname[TSDB_FILENAME_LEN]; char bname[TSDB_FILENAME_LEN];
STsdbFS * pfs = REPO_FS(pRepo); STsdbFS * pfs = REPO_FS(pRepo);
const TFILE *pf; const STfsFile *pf;
tsdbGetDataDir(REPO_ID(pRepo), dataDir); tsdbGetDataDir(REPO_ID(pRepo), dataDir);
TDIR *tdir = tfsOpendir(dataDir); STfsDir *tdir = tfsOpendir(pRepo->pTfs, dataDir);
if (tdir == NULL) { if (tdir == NULL) {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno));
return -1; return -1;
} }
while ((pf = tfsReaddir(tdir))) { while ((pf = tfsReaddir(tdir))) {
tfsbasename(pf, bname); tfsBasename(pf, bname);
if (!tsdbIsTFileInFS(pfs, pf)) { if (!tsdbIsTFileInFS(pfs, pf)) {
(void)tfsremove(pf); (void)tfsRemoveFile(pf);
tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), pf->aname);
} }
} }
...@@ -967,7 +968,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { ...@@ -967,7 +968,7 @@ static int tsdbScanDataDir(STsdb *pRepo) {
return 0; return 0;
} }
static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) {
SFSIter fsiter; SFSIter fsiter;
tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
SDFileSet *pSet; SDFileSet *pSet;
...@@ -987,8 +988,8 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { ...@@ -987,8 +988,8 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
// static int tsdbRestoreMeta(STsdb *pRepo) { // static int tsdbRestoreMeta(STsdb *pRepo) {
// char rootDir[TSDB_FILENAME_LEN]; // char rootDir[TSDB_FILENAME_LEN];
// char bname[TSDB_FILENAME_LEN]; // char bname[TSDB_FILENAME_LEN];
// TDIR * tdir = NULL; // STfsDir * tdir = NULL;
// const TFILE *pf = NULL; // const STfsFile *pf = NULL;
// const char * pattern = "^meta(-ver[0-9]+)?$"; // const char * pattern = "^meta(-ver[0-9]+)?$";
// regex_t regex; // regex_t regex;
// STsdbFS * pfs = REPO_FS(pRepo); // STsdbFS * pfs = REPO_FS(pRepo);
...@@ -1007,7 +1008,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { ...@@ -1007,7 +1008,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
// } // }
// while ((pf = tfsReaddir(tdir))) { // while ((pf = tfsReaddir(tdir))) {
// tfsbasename(pf, bname); // tfsBasename(pf, bname);
// if (strcmp(bname, "data") == 0) { // if (strcmp(bname, "data") == 0) {
// // Skip the data/ directory // // Skip the data/ directory
...@@ -1016,7 +1017,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { ...@@ -1016,7 +1017,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
// if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) { // if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) {
// // Skip current.t file // // Skip current.t file
// tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); // tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), pf->aname);
// (void)tfsremove(pf); // (void)tfsremove(pf);
// continue; // continue;
// } // }
...@@ -1026,7 +1027,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { ...@@ -1026,7 +1027,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
// // Match // // Match
// if (pfs->cstatus->pmf != NULL) { // if (pfs->cstatus->pmf != NULL) {
// tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo), // tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo),
// TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), TFILE_NAME(pf)); // TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), pf->aname);
// terrno = TSDB_CODE_TDB_FILE_CORRUPTED; // terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
// tfsClosedir(tdir); // tfsClosedir(tdir);
// regfree(&regex); // regfree(&regex);
...@@ -1081,7 +1082,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { ...@@ -1081,7 +1082,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
// } // }
// } else if (code == REG_NOMATCH) { // } else if (code == REG_NOMATCH) {
// // Not match // // Not match
// tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); // tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname);
// tfsremove(pf); // tfsremove(pf);
// continue; // continue;
// } else { // } else {
...@@ -1108,8 +1109,8 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { ...@@ -1108,8 +1109,8 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
static int tsdbRestoreDFileSet(STsdb *pRepo) { static int tsdbRestoreDFileSet(STsdb *pRepo) {
char dataDir[TSDB_FILENAME_LEN]; char dataDir[TSDB_FILENAME_LEN];
char bname[TSDB_FILENAME_LEN]; char bname[TSDB_FILENAME_LEN];
TDIR * tdir = NULL; STfsDir * tdir = NULL;
const TFILE *pf = NULL; const STfsFile *pf = NULL;
const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last)(-ver[0-9]+)?$"; const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last)(-ver[0-9]+)?$";
SArray * fArray = NULL; SArray * fArray = NULL;
regex_t regex; regex_t regex;
...@@ -1120,7 +1121,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { ...@@ -1120,7 +1121,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
// Resource allocation and init // Resource allocation and init
regcomp(&regex, pattern, REG_EXTENDED); regcomp(&regex, pattern, REG_EXTENDED);
fArray = taosArrayInit(1024, sizeof(TFILE)); fArray = taosArrayInit(1024, sizeof(STfsFile));
if (fArray == NULL) { if (fArray == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir,
...@@ -1129,7 +1130,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { ...@@ -1129,7 +1130,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
return -1; return -1;
} }
tdir = tfsOpendir(dataDir); tdir = tfsOpendir(pRepo->pTfs, dataDir);
if (tdir == NULL) { if (tdir == NULL) {
tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir,
tstrerror(terrno)); tstrerror(terrno));
...@@ -1139,7 +1140,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { ...@@ -1139,7 +1140,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
} }
while ((pf = tfsReaddir(tdir))) { while ((pf = tfsReaddir(tdir))) {
tfsbasename(pf, bname); tfsBasename(pf, bname);
int code = regexec(&regex, bname, 0, NULL, 0); int code = regexec(&regex, bname, 0, NULL, 0);
if (code == 0) { if (code == 0) {
...@@ -1152,8 +1153,8 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { ...@@ -1152,8 +1153,8 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
} }
} else if (code == REG_NOMATCH) { } else if (code == REG_NOMATCH) {
// Not match // Not match
tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname);
(void)tfsremove(pf); (void)tfsRemoveFile(pf);
continue; continue;
} else { } else {
// Has other error // Has other error
...@@ -1200,7 +1201,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { ...@@ -1200,7 +1201,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
uint32_t tversion; uint32_t tversion;
char _bname[TSDB_FILENAME_LEN]; char _bname[TSDB_FILENAME_LEN];
tfsbasename(pf, _bname); tfsBasename(pf, _bname);
tsdbParseDFilename(_bname, &tvid, &tfid, &ttype, &tversion); tsdbParseDFilename(_bname, &tvid, &tfid, &ttype, &tversion);
ASSERT(tvid == REPO_ID(pRepo)); ASSERT(tvid == REPO_ID(pRepo));
...@@ -1287,7 +1288,7 @@ static int tsdbRestoreCurrent(STsdb *pRepo) { ...@@ -1287,7 +1288,7 @@ static int tsdbRestoreCurrent(STsdb *pRepo) {
return -1; return -1;
} }
if (tsdbSaveFSStatus(pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) { if (tsdbSaveFSStatus(pRepo, pRepo->fs->cstatus) < 0) {
tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno)); tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1; return -1;
} }
...@@ -1296,8 +1297,8 @@ static int tsdbRestoreCurrent(STsdb *pRepo) { ...@@ -1296,8 +1297,8 @@ static int tsdbRestoreCurrent(STsdb *pRepo) {
} }
static int tsdbComparTFILE(const void *arg1, const void *arg2) { static int tsdbComparTFILE(const void *arg1, const void *arg2) {
TFILE *pf1 = (TFILE *)arg1; STfsFile *pf1 = (STfsFile *)arg1;
TFILE *pf2 = (TFILE *)arg2; STfsFile *pf2 = (STfsFile *)arg2;
int vid1, fid1, vid2, fid2; int vid1, fid1, vid2, fid2;
TSDB_FILE_T ftype1, ftype2; TSDB_FILE_T ftype1, ftype2;
...@@ -1305,8 +1306,8 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) { ...@@ -1305,8 +1306,8 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) {
char bname1[TSDB_FILENAME_LEN]; char bname1[TSDB_FILENAME_LEN];
char bname2[TSDB_FILENAME_LEN]; char bname2[TSDB_FILENAME_LEN];
tfsbasename(pf1, bname1); tfsBasename(pf1, bname1);
tfsbasename(pf2, bname2); tfsBasename(pf2, bname2);
tsdbParseDFilename(bname1, &vid1, &fid1, &ftype1, &version1); tsdbParseDFilename(bname1, &vid1, &fid1, &ftype1, &version1);
tsdbParseDFilename(bname2, &vid2, &fid2, &ftype2, &version2); tsdbParseDFilename(bname2, &vid2, &fid2, &ftype2, &version2);
......
source/dnode/vnode/src/tsdb/tsdbFile.c
浏览文件 @ f9d7cc44
...@@ -295,7 +295,7 @@ static int tsdbRollBackMFile(SMFile *pMFile) { ...@@ -295,7 +295,7 @@ static int tsdbRollBackMFile(SMFile *pMFile) {
#endif #endif
// ============== Operations on SDFile // ============== Operations on SDFile
void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) { void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype) {
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK); TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK);
...@@ -305,8 +305,8 @@ void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, ...@@ -305,8 +305,8 @@ void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver,
memset(&(pDFile->info), 0, sizeof(pDFile->info)); memset(&(pDFile->info), 0, sizeof(pDFile->info));
pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
tsdbGetFilename(vid, fid, ver, ftype, fname); tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname);
tfsInitFile(&(pDFile->f), did.level, did.id, fname); tfsInitFile(pRepo->pTfs, &(pDFile->f), did, fname);
} }
void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) { void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) {
...@@ -323,9 +323,9 @@ int tsdbEncodeSDFile(void **buf, SDFile *pDFile) { ...@@ -323,9 +323,9 @@ int tsdbEncodeSDFile(void **buf, SDFile *pDFile) {
return tlen; return tlen;
} }
void *tsdbDecodeSDFile(void *buf, SDFile *pDFile) { void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile) {
buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
buf = tfsDecodeFile(buf, &(pDFile->f)); buf = tfsDecodeFile(pRepo->pTfs, buf, &(pDFile->f));
TSDB_FILE_SET_CLOSED(pDFile); TSDB_FILE_SET_CLOSED(pDFile);
return buf; return buf;
...@@ -352,15 +352,15 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { ...@@ -352,15 +352,15 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
return buf; return buf;
} }
int tsdbCreateDFile(SDFile *pDFile, bool updateHeader) { int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pDFile->fd < 0) { if (pDFile->fd < 0) {
if (errno == ENOENT) { if (errno == ENOENT) {
// Try to create directory recursively // Try to create directory recursively
char *s = strdup(TFILE_REL_NAME(&(pDFile->f))); char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pDFile), TSDB_FILE_ID(pDFile)) < 0) { if (tfsMkdirRecurAt(pRepo->pTfs, dirname(s), TSDB_FILE_DID(pDFile)) < 0) {
tfree(s); tfree(s);
return -1; return -1;
} }
...@@ -559,13 +559,13 @@ static int tsdbRollBackDFile(SDFile *pDFile) { ...@@ -559,13 +559,13 @@ static int tsdbRollBackDFile(SDFile *pDFile) {
} }
// ============== Operations on SDFileSet // ============== Operations on SDFileSet
void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t ver) { void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver) {
pSet->fid = fid; pSet->fid = fid;
pSet->state = 0; pSet->state = 0;
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype);
tsdbInitDFile(pDFile, did, vid, fid, ver, ftype); tsdbInitDFile(pRepo, pDFile, did, fid, ver, ftype);
} }
} }
...@@ -587,14 +587,14 @@ int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) { ...@@ -587,14 +587,14 @@ int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) {
return tlen; return tlen;
} }
void *tsdbDecodeDFileSet(void *buf, SDFileSet *pSet) { void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) {
int32_t fid; int32_t fid;
buf = taosDecodeFixedI32(buf, &(fid)); buf = taosDecodeFixedI32(buf, &(fid));
pSet->state = 0; pSet->state = 0;
pSet->fid = fid; pSet->fid = fid;
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
buf = tsdbDecodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype)); buf = tsdbDecodeSDFile(pRepo, buf, TSDB_DFILE_IN_SET(pSet, ftype));
} }
return buf; return buf;
} }
...@@ -633,9 +633,9 @@ int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) { ...@@ -633,9 +633,9 @@ int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) {
return 0; return 0;
} }
int tsdbCreateDFileSet(SDFileSet *pSet, bool updateHeader) { int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
if (tsdbCreateDFile(TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) { if (tsdbCreateDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) {
tsdbCloseDFileSet(pSet); tsdbCloseDFileSet(pSet);
tsdbRemoveDFileSet(pSet); tsdbRemoveDFileSet(pSet);
return -1; return -1;
......
source/dnode/vnode/src/tsdb/tsdbMain.c
浏览文件 @ f9d7cc44
...@@ -16,12 +16,13 @@ ...@@ -16,12 +16,13 @@
#include "tsdbDef.h" #include "tsdbDef.h"
static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF,
SMeta *pMeta); SMeta *pMeta, STfs *pTfs);
static void tsdbFree(STsdb *pTsdb); static void tsdbFree(STsdb *pTsdb);
static int tsdbOpenImpl(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb);
static void tsdbCloseImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb);
STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta) { STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta,
STfs *pTfs) {
STsdb *pTsdb = NULL; STsdb *pTsdb = NULL;
// Set default TSDB Options // Set default TSDB Options
...@@ -36,7 +37,7 @@ STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAl ...@@ -36,7 +37,7 @@ STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAl
} }
// Create the handle // Create the handle
pTsdb = tsdbNew(path, vgId, pTsdbCfg, pMAF, pMeta); pTsdb = tsdbNew(path, vgId, pTsdbCfg, pMAF, pMeta, pTfs);
if (pTsdb == NULL) { if (pTsdb == NULL) {
// TODO: handle error // TODO: handle error
return NULL; return NULL;
...@@ -64,7 +65,7 @@ void tsdbRemove(const char *path) { taosRemoveDir(path); } ...@@ -64,7 +65,7 @@ void tsdbRemove(const char *path) { taosRemoveDir(path); }
/* ------------------------ STATIC METHODS ------------------------ */ /* ------------------------ STATIC METHODS ------------------------ */
static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF,
SMeta *pMeta) { SMeta *pMeta, STfs *pTfs) {
STsdb *pTsdb = NULL; STsdb *pTsdb = NULL;
pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); pTsdb = (STsdb *)calloc(1, sizeof(STsdb));
...@@ -78,6 +79,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, ...@@ -78,6 +79,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg); tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg);
pTsdb->pmaf = pMAF; pTsdb->pmaf = pMAF;
pTsdb->pMeta = pMeta; pTsdb->pMeta = pMeta;
pTsdb->pTfs = pTfs;
pTsdb->fs = tsdbNewFS(pTsdbCfg); pTsdb->fs = tsdbNewFS(pTsdbCfg);
...@@ -494,7 +496,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t ...@@ -494,7 +496,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t
} }
} else { // get the named file at the specified index. If not there, return 0 } else { // get the named file at the specified index. If not there, return 0
fname = malloc(256); fname = malloc(256);
sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name); sprintf(fname, "%s/vnode/vnode%d/%s", tfsGetPrimaryPath(pRepo->pTfs), REPO_ID(pRepo), name);
if (access(fname, F_OK) != 0) { if (access(fname, F_OK) != 0) {
tfree(fname); tfree(fname);
return 0; return 0;
......
source/dnode/vnode/src/vnd/vnodeMain.c
浏览文件 @ f9d7cc44
...@@ -28,6 +28,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { ...@@ -28,6 +28,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) {
if (pVnodeCfg != NULL) { if (pVnodeCfg != NULL) {
cfg.vgId = pVnodeCfg->vgId; cfg.vgId = pVnodeCfg->vgId;
cfg.pDnode = pVnodeCfg->pDnode; cfg.pDnode = pVnodeCfg->pDnode;
cfg.pTfs = pVnodeCfg->pTfs;
} }
// Validate options // Validate options
...@@ -75,6 +76,7 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { ...@@ -75,6 +76,7 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) {
pVnode->vgId = pVnodeCfg->vgId; pVnode->vgId = pVnodeCfg->vgId;
pVnode->pDnode = pVnodeCfg->pDnode; pVnode->pDnode = pVnodeCfg->pDnode;
pVnode->pTfs = pVnodeCfg->pTfs;
pVnode->path = strdup(path); pVnode->path = strdup(path);
vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg);
...@@ -109,7 +111,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { ...@@ -109,7 +111,7 @@ static int vnodeOpenImpl(SVnode *pVnode) {
// Open tsdb // Open tsdb
sprintf(dir, "%s/tsdb", pVnode->path); sprintf(dir, "%s/tsdb", pVnode->path);
pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta); pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs);
if (pVnode->pTsdb == NULL) { if (pVnode->pTsdb == NULL) {
// TODO: handle error // TODO: handle error
return -1; return -1;
......
source/dnode/vnode/src/vnd/vnodeWrite.c
浏览文件 @ f9d7cc44
...@@ -108,6 +108,15 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ...@@ -108,6 +108,15 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
// TODO: handle error // TODO: handle error
} }
break; break;
case TDMT_VND_MQ_SET_CONN: {
char* reqStr = ptr;
SMqSetCVgReq req;
/*tDecodeSMqSetCVgReq(reqStr, &req);*/
// create topic if not exist
// convert to task
// write mq meta
}
break;
default: default:
ASSERT(0); ASSERT(0);
break; break;
......
source/libs/scheduler/CMakeLists.txt
浏览文件 @ f9d7cc44
...@@ -9,9 +9,9 @@ target_include_directories( ...@@ -9,9 +9,9 @@ target_include_directories(
target_link_libraries( target_link_libraries(
scheduler scheduler
PRIVATE os util planner qcom common catalog transport PUBLIC os util planner qcom common catalog transport
) )
if(${BUILD_TEST}) if(${BUILD_TEST})
ADD_SUBDIRECTORY(test) ADD_SUBDIRECTORY(test)
endif(${BUILD_TEST}) endif(${BUILD_TEST})
\ No newline at end of file
source/libs/scheduler/src/scheduler.c
浏览文件 @ f9d7cc44
...@@ -13,19 +13,42 @@ ...@@ -13,19 +13,42 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "catalog.h"
#include "query.h"
#include "schedulerInt.h" #include "schedulerInt.h"
#include "tmsg.h" #include "tmsg.h"
#include "query.h"
#include "catalog.h"
static SSchedulerMgmt schMgmt = {0}; static SSchedulerMgmt schMgmt = {0};
uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); }
uint64_t schGenUUID(void) {
static uint64_t hashId = 0;
static int32_t requestSerialId = 0;
if (hashId == 0) {
char uid[64];
int32_t code = taosGetSystemUUID(uid, tListLen(uid));
if (code != TSDB_CODE_SUCCESS) {
qError("Failed to get the system uid, reason:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
} else {
hashId = MurmurHash3_32(uid, strlen(uid));
}
}
int64_t ts = taosGetTimestampMs();
uint64_t pid = taosGetPId();
int32_t val = atomic_add_fetch_32(&requestSerialId, 1);
uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF);
return id;
}
int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel *pLevel) { int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) {
pTask->plan = pPlan; pTask->plan = pPlan;
pTask->level = pLevel; pTask->level = pLevel;
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START);
pTask->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); pTask->taskId = schGenTaskId();
pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr));
if (NULL == pTask->execAddrs) { if (NULL == pTask->execAddrs) {
SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CANDIDATE_EP_NUM); SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CANDIDATE_EP_NUM);
...@@ -35,13 +58,12 @@ int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel * ...@@ -35,13 +58,12 @@ int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel *
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void schFreeTask(SSchTask* pTask) { void schFreeTask(SSchTask *pTask) {
if (pTask->candidateAddrs) { if (pTask->candidateAddrs) {
taosArrayDestroy(pTask->candidateAddrs); taosArrayDestroy(pTask->candidateAddrs);
} }
// TODO NEED TO VERFY WITH ASYNC_SEND MEMORY FREE tfree(pTask->msg);
//tfree(pTask->msg);
if (pTask->children) { if (pTask->children) {
taosArrayDestroy(pTask->children); taosArrayDestroy(pTask->children);
...@@ -56,10 +78,9 @@ void schFreeTask(SSchTask* pTask) { ...@@ -56,10 +78,9 @@ void schFreeTask(SSchTask* pTask) {
} }
} }
int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) {
int32_t lastMsgType = atomic_load_32(&pTask->lastMsgType); int32_t lastMsgType = atomic_load_32(&pTask->lastMsgType);
switch (msgType) { switch (msgType) {
case TDMT_VND_CREATE_TABLE_RSP: case TDMT_VND_CREATE_TABLE_RSP:
case TDMT_VND_SUBMIT_RSP: case TDMT_VND_SUBMIT_RSP:
...@@ -71,23 +92,24 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m ...@@ -71,23 +92,24 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m
SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%d, rspType:%d", lastMsgType, msgType); SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%d, rspType:%d", lastMsgType, msgType);
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING &&
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType); SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask),
msgType);
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
break; break;
default: default:
SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask));
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) {
int32_t code = 0; int32_t code = 0;
...@@ -99,37 +121,34 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { ...@@ -99,37 +121,34 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) {
if (oriStatus == newStatus) { if (oriStatus == newStatus) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
switch (oriStatus) { switch (oriStatus) {
case JOB_TASK_STATUS_NULL: case JOB_TASK_STATUS_NULL:
if (newStatus != JOB_TASK_STATUS_NOT_START) { if (newStatus != JOB_TASK_STATUS_NOT_START) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break; break;
case JOB_TASK_STATUS_NOT_START: case JOB_TASK_STATUS_NOT_START:
if (newStatus != JOB_TASK_STATUS_EXECUTING) { if (newStatus != JOB_TASK_STATUS_EXECUTING) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break; break;
case JOB_TASK_STATUS_EXECUTING: case JOB_TASK_STATUS_EXECUTING:
if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED && newStatus != JOB_TASK_STATUS_FAILED &&
&& newStatus != JOB_TASK_STATUS_FAILED newStatus != JOB_TASK_STATUS_CANCELLING && newStatus != JOB_TASK_STATUS_CANCELLED &&
&& newStatus != JOB_TASK_STATUS_CANCELLING newStatus != JOB_TASK_STATUS_DROPPING) {
&& newStatus != JOB_TASK_STATUS_CANCELLED
&& newStatus != JOB_TASK_STATUS_DROPPING) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break; break;
case JOB_TASK_STATUS_PARTIAL_SUCCEED: case JOB_TASK_STATUS_PARTIAL_SUCCEED:
if (newStatus != JOB_TASK_STATUS_FAILED if (newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_SUCCEED &&
&& newStatus != JOB_TASK_STATUS_SUCCEED newStatus != JOB_TASK_STATUS_DROPPING) {
&& newStatus != JOB_TASK_STATUS_DROPPING) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break; break;
case JOB_TASK_STATUS_SUCCEED: case JOB_TASK_STATUS_SUCCEED:
case JOB_TASK_STATUS_FAILED: case JOB_TASK_STATUS_FAILED:
...@@ -137,13 +156,13 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { ...@@ -137,13 +156,13 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) {
if (newStatus != JOB_TASK_STATUS_DROPPING) { if (newStatus != JOB_TASK_STATUS_DROPPING) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break; break;
case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_CANCELLED:
case JOB_TASK_STATUS_DROPPING: case JOB_TASK_STATUS_DROPPING:
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED);
break; break;
default: default:
SCH_JOB_ELOG("invalid job status:%d", oriStatus); SCH_JOB_ELOG("invalid job status:%d", oriStatus);
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
...@@ -166,23 +185,22 @@ _return: ...@@ -166,23 +185,22 @@ _return:
SCH_ERR_RET(code); SCH_ERR_RET(code);
} }
int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t i = 0; i < pJob->levelNum; ++i) { for (int32_t i = 0; i < pJob->levelNum; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i); SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
for (int32_t m = 0; m < pLevel->taskNum; ++m) { for (int32_t m = 0; m < pLevel->taskNum; ++m) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); SSchTask *pTask = taosArrayGet(pLevel->subTasks, m);
SSubplan *pPlan = pTask->plan; SSubplan *pPlan = pTask->plan;
int32_t childNum = pPlan->pChildren ? (int32_t)taosArrayGetSize(pPlan->pChildren) : 0; int32_t childNum = pPlan->pChildren ? (int32_t)taosArrayGetSize(pPlan->pChildren) : 0;
int32_t parentNum = pPlan->pParents ? (int32_t)taosArrayGetSize(pPlan->pParents) : 0; int32_t parentNum = pPlan->pParents ? (int32_t)taosArrayGetSize(pPlan->pParents) : 0;
if (childNum > 0) { if (childNum > 0) {
if (pJob->levelIdx == pLevel->level) { if (pJob->levelIdx == pLevel->level) {
SCH_JOB_ELOG("invalid query plan, lowest level, childNum:%d", childNum); SCH_JOB_ELOG("invalid query plan, lowest level, childNum:%d", childNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
pTask->children = taosArrayInit(childNum, POINTER_BYTES); pTask->children = taosArrayInit(childNum, POINTER_BYTES);
if (NULL == pTask->children) { if (NULL == pTask->children) {
SCH_TASK_ELOG("taosArrayInit %d children failed", childNum); SCH_TASK_ELOG("taosArrayInit %d children failed", childNum);
...@@ -209,7 +227,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { ...@@ -209,7 +227,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
SCH_TASK_ELOG("invalid task info, level:0, parentNum:%d", parentNum); SCH_TASK_ELOG("invalid task info, level:0, parentNum:%d", parentNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
pTask->parents = taosArrayInit(parentNum, POINTER_BYTES); pTask->parents = taosArrayInit(parentNum, POINTER_BYTES);
if (NULL == pTask->parents) { if (NULL == pTask->parents) {
SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum); SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum);
...@@ -234,7 +252,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { ...@@ -234,7 +252,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
} }
SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum);
} }
...@@ -249,7 +267,6 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { ...@@ -249,7 +267,6 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schRecordTaskSucceedNode(SSchTask *pTask) { int32_t schRecordTaskSucceedNode(SSchTask *pTask) {
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx));
...@@ -260,7 +277,6 @@ int32_t schRecordTaskSucceedNode(SSchTask *pTask) { ...@@ -260,7 +277,6 @@ int32_t schRecordTaskSucceedNode(SSchTask *pTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr) { int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr) {
if (NULL == taosArrayPush(pTask->execAddrs, addr)) { if (NULL == taosArrayPush(pTask->execAddrs, addr)) {
SCH_TASK_ELOG("taosArrayPush addr to execAddr list failed, errno:%d", errno); SCH_TASK_ELOG("taosArrayPush addr to execAddr list failed, errno:%d", errno);
...@@ -270,23 +286,25 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad ...@@ -270,23 +286,25 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) {
int32_t code = 0; int32_t code = 0;
pJob->queryId = pDag->queryId; pJob->queryId = pDag->queryId;
if (pDag->numOfSubplans <= 0) { if (pDag->numOfSubplans <= 0) {
SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans);
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
int32_t levelNum = (int32_t)taosArrayGetSize(pDag->pSubplans); int32_t levelNum = (int32_t)taosArrayGetSize(pDag->pSubplans);
if (levelNum <= 0) { if (levelNum <= 0) {
SCH_JOB_ELOG("invalid level num:%d", levelNum); SCH_JOB_ELOG("invalid level num:%d", levelNum);
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
SHashObj *planToTask = taosHashInit(SCHEDULE_DEFAULT_TASK_NUMBER, taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); SHashObj *planToTask = taosHashInit(
SCHEDULE_DEFAULT_TASK_NUMBER,
taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false,
HASH_NO_LOCK);
if (NULL == planToTask) { if (NULL == planToTask) {
SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_TASK_NUMBER); SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_TASK_NUMBER);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
...@@ -303,9 +321,9 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { ...@@ -303,9 +321,9 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) {
pJob->subPlans = pDag->pSubplans; pJob->subPlans = pDag->pSubplans;
SSchLevel level = {0}; SSchLevel level = {0};
SArray *plans = NULL; SArray * plans = NULL;
int32_t taskNum = 0; int32_t taskNum = 0;
SSchLevel *pLevel = NULL; SSchLevel *pLevel = NULL;
level.status = JOB_TASK_STATUS_NOT_START; level.status = JOB_TASK_STATUS_NOT_START;
...@@ -317,9 +335,9 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { ...@@ -317,9 +335,9 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) {
} }
pLevel = taosArrayGet(pJob->levels, i); pLevel = taosArrayGet(pJob->levels, i);
pLevel->level = i; pLevel->level = i;
plans = taosArrayGetP(pDag->pSubplans, i); plans = taosArrayGetP(pDag->pSubplans, i);
if (NULL == plans) { if (NULL == plans) {
SCH_JOB_ELOG("empty level plan, level:%d", i); SCH_JOB_ELOG("empty level plan, level:%d", i);
...@@ -333,13 +351,13 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { ...@@ -333,13 +351,13 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) {
} }
pLevel->taskNum = taskNum; pLevel->taskNum = taskNum;
pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask)); pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask));
if (NULL == pLevel->subTasks) { if (NULL == pLevel->subTasks) {
SCH_JOB_ELOG("taosArrayInit %d failed", taskNum); SCH_JOB_ELOG("taosArrayInit %d failed", taskNum);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
for (int32_t n = 0; n < taskNum; ++n) { for (int32_t n = 0; n < taskNum; ++n) {
SSubplan *plan = taosArrayGetP(plans, n); SSubplan *plan = taosArrayGetP(plans, n);
...@@ -347,15 +365,15 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { ...@@ -347,15 +365,15 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) {
SSchTask task = {0}; SSchTask task = {0};
SSchTask *pTask = &task; SSchTask *pTask = &task;
SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel)); SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel));
void *p = taosArrayPush(pLevel->subTasks, &task); void *p = taosArrayPush(pLevel->subTasks, &task);
if (NULL == p) { if (NULL == p) {
SCH_TASK_ELOG("taosArrayPush task to level failed, level:%d, taskIdx:%d", pLevel->level, n); SCH_TASK_ELOG("taosArrayPush task to level failed, level:%d, taskIdx:%d", pLevel->level, n);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) {
SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
...@@ -404,10 +422,10 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { ...@@ -404,10 +422,10 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
int32_t nodeNum = 0; int32_t nodeNum = 0;
if (pJob->nodeList) { if (pJob->nodeList) {
nodeNum = taosArrayGetSize(pJob->nodeList); nodeNum = taosArrayGetSize(pJob->nodeList);
for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) {
SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i); SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i);
if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) {
SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
...@@ -422,14 +440,14 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { ...@@ -422,14 +440,14 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
return TSDB_CODE_QRY_INVALID_INPUT; return TSDB_CODE_QRY_INVALID_INPUT;
} }
/* /*
for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) {
strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i]));
epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i];
++epSet->numOfEps; ++epSet->numOfEps;
} }
*/ */
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -441,7 +459,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { ...@@ -441,7 +459,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) {
SCH_TASK_ELOG("task already in execTask list, code:%x", code); SCH_TASK_ELOG("task already in execTask list, code:%x", code);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
...@@ -460,11 +478,11 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { ...@@ -460,11 +478,11 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
if (0 != code) { if (0 != code) {
if (HASH_NODE_EXIST(code)) { if (HASH_NODE_EXIST(code)) {
*moved = true; *moved = true;
SCH_TASK_ELOG("task already in succTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_TASK_ELOG("task already in succTask list, status:%d", SCH_GET_TASK_STATUS(pTask));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
...@@ -472,13 +490,13 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { ...@@ -472,13 +490,13 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
*moved = true; *moved = true;
SCH_TASK_DLOG("task moved to succTask list, numOfTasks:%d", taosHashGetSize(pJob->succTasks)); SCH_TASK_DLOG("task moved to succTask list, numOfTasks:%d", taosHashGetSize(pJob->succTasks));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
*moved = false; *moved = false;
if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) {
SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask));
} }
...@@ -487,11 +505,11 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { ...@@ -487,11 +505,11 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
if (0 != code) { if (0 != code) {
if (HASH_NODE_EXIST(code)) { if (HASH_NODE_EXIST(code)) {
*moved = true; *moved = true;
SCH_TASK_WLOG("task already in failTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_TASK_WLOG("task already in failTask list, status:%d", SCH_GET_TASK_STATUS(pTask));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
...@@ -499,11 +517,10 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { ...@@ -499,11 +517,10 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
*moved = true; *moved = true;
SCH_TASK_DLOG("task moved to failTask list, numOfTasks:%d", taosHashGetSize(pJob->failTasks)); SCH_TASK_DLOG("task moved to failTask list, numOfTasks:%d", taosHashGetSize(pJob->failTasks));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) { if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) {
SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask));
...@@ -513,11 +530,11 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { ...@@ -513,11 +530,11 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
if (0 != code) { if (0 != code) {
if (HASH_NODE_EXIST(code)) { if (HASH_NODE_EXIST(code)) {
*moved = true; *moved = true;
SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
...@@ -525,11 +542,10 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { ...@@ -525,11 +542,10 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
*moved = true; *moved = true;
SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) {
// TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO set retry or not based on task type/errCode/retry times/job status/available eps...
// TODO if needRetry, set task retry info // TODO if needRetry, set task retry info
...@@ -541,13 +557,10 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b ...@@ -541,13 +557,10 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCode) {
// Note: no more error processing, handled in function internal
int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) {
// if already FAILED, no more processing // if already FAILED, no more processing
SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, status));
if (errCode) { if (errCode) {
atomic_store_32(&pJob->errCode, errCode); atomic_store_32(&pJob->errCode, errCode);
} }
...@@ -561,12 +574,20 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { ...@@ -561,12 +574,20 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) {
assert(0); assert(0);
} }
// Note: no more error processing, handled in function internal
int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) {
SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_FAILED, errCode));
}
// Note: no more error processing, handled in function internal
int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) {
SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_DROPPING, errCode));
}
// Note: no more error processing, handled in function internal // Note: no more error processing, handled in function internal
int32_t schFetchFromRemote(SSchJob *pJob) { int32_t schFetchFromRemote(SSchJob *pJob) {
int32_t code = 0; int32_t code = 0;
if (atomic_val_compare_exchange_32(&pJob->remoteFetch, 0, 1) != 0) { if (atomic_val_compare_exchange_32(&pJob->remoteFetch, 0, 1) != 0) {
SCH_JOB_ELOG("prior fetching not finished, remoteFetch:%d", atomic_load_32(&pJob->remoteFetch)); SCH_JOB_ELOG("prior fetching not finished, remoteFetch:%d", atomic_load_32(&pJob->remoteFetch));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -583,7 +604,7 @@ int32_t schFetchFromRemote(SSchJob *pJob) { ...@@ -583,7 +604,7 @@ int32_t schFetchFromRemote(SSchJob *pJob) {
SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0);
...@@ -593,17 +614,16 @@ _return: ...@@ -593,17 +614,16 @@ _return:
return code; return code;
} }
// Note: no more error processing, handled in function internal // Note: no more error processing, handled in function internal
int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) {
int32_t code = 0; int32_t code = 0;
SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED));
if ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule) { if ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule) {
tsem_post(&pJob->rspSem); tsem_post(&pJob->rspSem);
} }
if (atomic_load_8(&pJob->userFetch)) { if (atomic_load_8(&pJob->userFetch)) {
SCH_ERR_JRET(schFetchFromRemote(pJob)); SCH_ERR_JRET(schFetchFromRemote(pJob));
} }
...@@ -617,26 +637,23 @@ _return: ...@@ -617,26 +637,23 @@ _return:
SCH_RET(code); SCH_RET(code);
} }
int32_t schProcessOnDataFetched(SSchJob *job) { int32_t schProcessOnDataFetched(SSchJob *job) {
atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0);
tsem_post(&job->rspSem); tsem_post(&job->rspSem);
} }
// Note: no more error processing, handled in function internal // Note: no more error processing, handled in function internal
int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) {
bool needRetry = false; bool needRetry = false;
bool moved = false; bool moved = false;
int32_t taskDone = 0; int32_t taskDone = 0;
int32_t code = 0; int32_t code = 0;
SCH_TASK_DLOG("taskOnFailure, code:%x", errCode); SCH_TASK_DLOG("taskOnFailure, code:%x", errCode);
SCH_ERR_JRET(schTaskCheckAndSetRetry(pJob, pTask, errCode, &needRetry)); SCH_ERR_JRET(schTaskCheckAndSetRetry(pJob, pTask, errCode, &needRetry));
if (!needRetry) { if (!needRetry) {
SCH_TASK_ELOG("task failed and no more retry, code:%x", errCode); SCH_TASK_ELOG("task failed and no more retry, code:%x", errCode);
...@@ -648,7 +665,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) ...@@ -648,7 +665,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode)
} }
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED);
if (SCH_TASK_NEED_WAIT_ALL(pTask)) { if (SCH_TASK_NEED_WAIT_ALL(pTask)) {
SCH_LOCK(SCH_WRITE, &pTask->level->lock); SCH_LOCK(SCH_WRITE, &pTask->level->lock);
pTask->level->taskFailed++; pTask->level->taskFailed++;
...@@ -656,7 +673,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) ...@@ -656,7 +673,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode)
SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); SCH_UNLOCK(SCH_WRITE, &pTask->level->lock);
atomic_store_32(&pJob->errCode, errCode); atomic_store_32(&pJob->errCode, errCode);
if (taskDone < pTask->level->taskNum) { if (taskDone < pTask->level->taskNum) {
SCH_TASK_DLOG("not all tasks done, done:%d, all:%d", taskDone, pTask->level->taskNum); SCH_TASK_DLOG("not all tasks done, done:%d, all:%d", taskDone, pTask->level->taskNum);
SCH_ERR_RET(errCode); SCH_ERR_RET(errCode);
...@@ -665,7 +682,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) ...@@ -665,7 +682,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode)
} else { } else {
// Note: no more error processing, already handled // Note: no more error processing, already handled
SCH_ERR_RET(schLaunchTask(pJob, pTask)); SCH_ERR_RET(schLaunchTask(pJob, pTask));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -676,15 +693,14 @@ _return: ...@@ -676,15 +693,14 @@ _return:
SCH_ERR_RET(errCode); SCH_ERR_RET(errCode);
} }
// Note: no more error processing, handled in function internal // Note: no more error processing, handled in function internal
int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
bool moved = false; bool moved = false;
int32_t code = 0; int32_t code = 0;
SSchTask *pErrTask = pTask; SSchTask *pErrTask = pTask;
SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask));
code = schMoveTaskToSuccList(pJob, pTask, &moved); code = schMoveTaskToSuccList(pJob, pTask, &moved);
if (code && moved) { if (code && moved) {
SCH_ERR_RET(code); SCH_ERR_RET(code);
...@@ -693,20 +709,20 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { ...@@ -693,20 +709,20 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED);
SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); SCH_ERR_JRET(schRecordTaskSucceedNode(pTask));
int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0;
if (parentNum == 0) { if (parentNum == 0) {
int32_t taskDone = 0; int32_t taskDone = 0;
if (SCH_TASK_NEED_WAIT_ALL(pTask)) { if (SCH_TASK_NEED_WAIT_ALL(pTask)) {
SCH_LOCK(SCH_WRITE, &pTask->level->lock); SCH_LOCK(SCH_WRITE, &pTask->level->lock);
pTask->level->taskSucceed++; pTask->level->taskSucceed++;
taskDone = pTask->level->taskSucceed + pTask->level->taskFailed; taskDone = pTask->level->taskSucceed + pTask->level->taskFailed;
SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); SCH_UNLOCK(SCH_WRITE, &pTask->level->lock);
if (taskDone < pTask->level->taskNum) { if (taskDone < pTask->level->taskNum) {
SCH_TASK_ELOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); SCH_TASK_ELOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (taskDone > pTask->level->taskNum) { } else if (taskDone > pTask->level->taskNum) {
assert(0); assert(0);
...@@ -727,31 +743,31 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { ...@@ -727,31 +743,31 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
if (code && moved) { if (code && moved) {
SCH_ERR_RET(code); SCH_ERR_RET(code);
} }
SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob)); SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
/* /*
if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) {
strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn));
job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port;
++job->dataSrcEps.numOfEps; ++job->dataSrcEps.numOfEps;
} }
*/ */
for (int32_t i = 0; i < parentNum; ++i) { for (int32_t i = 0; i < parentNum; ++i) {
SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i);
pErrTask = par; pErrTask = par;
atomic_add_fetch_32(&par->childReady, 1); atomic_add_fetch_32(&par->childReady, 1);
SCH_LOCK(SCH_WRITE, &par->lock); SCH_LOCK(SCH_WRITE, &par->lock);
qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->succeedAddr); qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->succeedAddr);
SCH_UNLOCK(SCH_WRITE, &par->lock); SCH_UNLOCK(SCH_WRITE, &par->lock);
if (SCH_TASK_READY_TO_LUNCH(par)) { if (SCH_TASK_READY_TO_LUNCH(par)) {
SCH_ERR_RET(schLaunchTask(pJob, par)); SCH_ERR_RET(schLaunchTask(pJob, par));
} }
...@@ -766,23 +782,24 @@ _return: ...@@ -766,23 +782,24 @@ _return:
SCH_ERR_RET(code); SCH_ERR_RET(code);
} }
int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize,
int32_t rspCode) {
int32_t code = 0; int32_t code = 0;
SCH_ERR_JRET(schValidateTaskReceivedMsgType(pJob, pTask, msgType)); SCH_ERR_JRET(schValidateTaskReceivedMsgType(pJob, pTask, msgType));
switch (msgType) { switch (msgType) {
case TDMT_VND_CREATE_TABLE_RSP: { case TDMT_VND_CREATE_TABLE_RSP: {
if (rspCode != TSDB_CODE_SUCCESS) { if (rspCode != TSDB_CODE_SUCCESS) {
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode));
}
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
} }
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
}
case TDMT_VND_SUBMIT_RSP: { case TDMT_VND_SUBMIT_RSP: {
#if 0 //TODO OPEN THIS #if 0 // TODO OPEN THIS
SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg;
if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) {
...@@ -790,65 +807,75 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch ...@@ -790,65 +807,75 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
} }
pJob->resNumOfRows += rsp->affectedRows; pJob->resNumOfRows += rsp->affectedRows;
#else #else
if (rspCode != TSDB_CODE_SUCCESS) { if (rspCode != TSDB_CODE_SUCCESS) {
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode));
} }
SShellSubmitRsp *rsp = (SShellSubmitRsp *)msg; SShellSubmitRsp *rsp = (SShellSubmitRsp *)msg;
if (rsp) { if (rsp) {
pJob->resNumOfRows += rsp->affectedRows; pJob->resNumOfRows += rsp->affectedRows;
} }
#endif #endif
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break; break;
} }
case TDMT_VND_QUERY_RSP: { case TDMT_VND_QUERY_RSP: {
SQueryTableRsp *rsp = (SQueryTableRsp *)msg; SQueryTableRsp *rsp = (SQueryTableRsp *)msg;
if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) {
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode));
}
SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY));
break;
} }
SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY));
break;
}
case TDMT_VND_RES_READY_RSP: { case TDMT_VND_RES_READY_RSP: {
SResReadyRsp *rsp = (SResReadyRsp *)msg; SResReadyRsp *rsp = (SResReadyRsp *)msg;
if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) {
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code)); SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code));
}
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
} }
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
}
case TDMT_VND_FETCH_RSP: { case TDMT_VND_FETCH_RSP: {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg;
if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) {
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode));
} }
atomic_store_ptr(&pJob->res, rsp); if (pJob->res) {
atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows); SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->res);
tfree(rsp);
SCH_ERR_JRET(schProcessOnDataFetched(pJob)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
break;
} }
case TDMT_VND_DROP_TASK: {
// SHOULD NEVER REACH HERE atomic_store_ptr(&pJob->res, rsp);
assert(0); atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows);
break;
if (rsp->completed) {
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED);
} }
SCH_ERR_JRET(schProcessOnDataFetched(pJob));
break;
}
case TDMT_VND_DROP_TASK: {
// SHOULD NEVER REACH HERE
assert(0);
break;
}
default: default:
SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask));
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
} }
...@@ -857,21 +884,20 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch ...@@ -857,21 +884,20 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
_return: _return:
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code)); SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code));
SCH_RET(code); SCH_RET(code);
} }
int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) {
int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0;
int32_t code = 0;
SSchCallbackParam *pParam = (SSchCallbackParam *)param; SSchCallbackParam *pParam = (SSchCallbackParam *)param;
SSchJob *pJob = NULL; SSchJob * pJob = NULL;
SSchTask *pTask = NULL; SSchTask * pTask = NULL;
SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId));
if (NULL == job || NULL == (*job)) { if (NULL == job || NULL == (*job)) {
qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId); qError("QID:%" PRIx64 " taosHashGet queryId not exist, may be dropped", pParam->queryId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED);
} }
pJob = *job; pJob = *job;
...@@ -880,20 +906,20 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in ...@@ -880,20 +906,20 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in
int32_t s = taosHashGetSize(pJob->execTasks); int32_t s = taosHashGetSize(pJob->execTasks);
if (s <= 0) { if (s <= 0) {
qError("QID:%"PRIx64",TID:%"PRIx64" no task in execTask list", pParam->queryId, pParam->taskId); qError("QID:%" PRIx64 ",TID:%" PRId64 " no task in execTask list", pParam->queryId, pParam->taskId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId));
if (NULL == task || NULL == (*task)) { if (NULL == task || NULL == (*task)) {
qError("QID:%"PRIx64",TID:%"PRIx64" taosHashGet taskId not exist", pParam->queryId, pParam->taskId); qError("QID:%" PRIx64 ",TID:%" PRId64 " taosHashGet taskId not exist", pParam->queryId, pParam->taskId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
pTask = *task; pTask = *task;
SCH_TASK_DLOG("rsp msg received, type:%d, code:%x", msgType, rspCode); SCH_TASK_DLOG("rsp msg received, type:%d, code:%x", msgType, rspCode);
SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode));
_return: _return:
...@@ -906,29 +932,29 @@ _return: ...@@ -906,29 +932,29 @@ _return:
SCH_RET(code); SCH_RET(code);
} }
int32_t schHandleSubmitCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t schHandleSubmitCallback(void *param, const SDataBuf *pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_SUBMIT_RSP, code); return schHandleCallback(param, pMsg, TDMT_VND_SUBMIT_RSP, code);
} }
int32_t schHandleCreateTableCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t schHandleCreateTableCallback(void *param, const SDataBuf *pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_CREATE_TABLE_RSP, code); return schHandleCallback(param, pMsg, TDMT_VND_CREATE_TABLE_RSP, code);
} }
int32_t schHandleQueryCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t schHandleQueryCallback(void *param, const SDataBuf *pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code); return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code);
} }
int32_t schHandleFetchCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t schHandleFetchCallback(void *param, const SDataBuf *pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_FETCH_RSP, code); return schHandleCallback(param, pMsg, TDMT_VND_FETCH_RSP, code);
} }
int32_t schHandleReadyCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t schHandleReadyCallback(void *param, const SDataBuf *pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code); return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code);
} }
int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) {
SSchCallbackParam *pParam = (SSchCallbackParam *)param; SSchCallbackParam *pParam = (SSchCallbackParam *)param;
qDebug("QID:%"PRIx64",TID:%"PRIx64" drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code);
} }
int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
...@@ -936,16 +962,16 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { ...@@ -936,16 +962,16 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
case TDMT_VND_CREATE_TABLE: case TDMT_VND_CREATE_TABLE:
*fp = schHandleCreateTableCallback; *fp = schHandleCreateTableCallback;
break; break;
case TDMT_VND_SUBMIT: case TDMT_VND_SUBMIT:
*fp = schHandleSubmitCallback; *fp = schHandleSubmitCallback;
break; break;
case TDMT_VND_QUERY: case TDMT_VND_QUERY:
*fp = schHandleQueryCallback; *fp = schHandleQueryCallback;
break; break;
case TDMT_VND_RES_READY: case TDMT_VND_RES_READY:
*fp = schHandleReadyCallback; *fp = schHandleReadyCallback;
break; break;
case TDMT_VND_FETCH: case TDMT_VND_FETCH:
*fp = schHandleFetchCallback; *fp = schHandleFetchCallback;
break; break;
case TDMT_VND_DROP_TASK: case TDMT_VND_DROP_TASK:
...@@ -959,18 +985,18 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { ...@@ -959,18 +985,18 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schAsyncSendMsg(void *transport, SEpSet *epSet, uint64_t qId, uint64_t tId, int32_t msgType, void *msg,
int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t tId, int32_t msgType, void *msg, uint32_t msgSize) { uint32_t msgSize) {
int32_t code = 0; int32_t code = 0;
SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); SMsgSendInfo *pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) { if (NULL == pMsgSendInfo) {
qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SMsgSendInfo)); qError("QID:%" PRIx64 ",TID:%" PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SMsgSendInfo));
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam)); SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam));
if (NULL == param) { if (NULL == param) {
qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SSchCallbackParam)); qError("QID:%" PRIx64 ",TID:%" PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SSchCallbackParam));
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
...@@ -985,19 +1011,19 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t ...@@ -985,19 +1011,19 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t
pMsgSendInfo->msgInfo.len = msgSize; pMsgSendInfo->msgInfo.len = msgSize;
pMsgSendInfo->msgType = msgType; pMsgSendInfo->msgType = msgType;
pMsgSendInfo->fp = fp; pMsgSendInfo->fp = fp;
int64_t transporterId = 0; int64_t transporterId = 0;
code = asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo); code = asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo);
if (code) { if (code) {
qError("QID:%"PRIx64 ",TID:%"PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); qError("QID:%" PRIx64 ",TID:%" PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code);
SCH_ERR_JRET(code); SCH_ERR_JRET(code);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
tfree(param); tfree(param);
tfree(pMsgSendInfo); tfree(pMsgSendInfo);
...@@ -1007,7 +1033,7 @@ _return: ...@@ -1007,7 +1033,7 @@ _return:
void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) { void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) {
epSet->inUse = addr->inUse; epSet->inUse = addr->inUse;
epSet->numOfEps = addr->numOfEps; epSet->numOfEps = addr->numOfEps;
for (int8_t i = 0; i < epSet->numOfEps; ++i) { for (int8_t i = 0; i < epSet->numOfEps; ++i) {
strncpy(epSet->fqdn[i], addr->epAddr[i].fqdn, sizeof(addr->epAddr[i].fqdn)); strncpy(epSet->fqdn[i], addr->epAddr[i].fqdn, sizeof(addr->epAddr[i].fqdn));
epSet->port[i] = addr->epAddr[i].port; epSet->port[i] = addr->epAddr[i].port;
...@@ -1016,24 +1042,30 @@ void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) { ...@@ -1016,24 +1042,30 @@ void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) {
int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) {
uint32_t msgSize = 0; uint32_t msgSize = 0;
void *msg = NULL; void * msg = NULL;
int32_t code = 0; int32_t code = 0;
bool isCandidateAddr = false; bool isCandidateAddr = false;
SEpSet epSet; SEpSet epSet;
if (NULL == addr) { if (NULL == addr) {
addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx));
isCandidateAddr = true; isCandidateAddr = true;
} }
schConvertAddrToEpSet(addr, &epSet); schConvertAddrToEpSet(addr, &epSet);
switch (msgType) { switch (msgType) {
case TDMT_VND_CREATE_TABLE: case TDMT_VND_CREATE_TABLE:
case TDMT_VND_SUBMIT: { case TDMT_VND_SUBMIT: {
msgSize = pTask->msgLen; msgSize = pTask->msgLen;
msg = pTask->msg; msg = calloc(1, msgSize);
if (NULL == msg) {
SCH_TASK_ELOG("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
memcpy(msg, pTask->msg, msgSize);
break; break;
} }
...@@ -1048,7 +1080,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, ...@@ -1048,7 +1080,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
SSubQueryMsg *pMsg = msg; SSubQueryMsg *pMsg = msg;
pMsg->header.vgId = htonl(addr->nodeId); pMsg->header.vgId = htonl(addr->nodeId);
pMsg->sId = htobe64(schMgmt.sId); pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(pJob->queryId); pMsg->queryId = htobe64(pJob->queryId);
pMsg->taskId = htobe64(pTask->taskId); pMsg->taskId = htobe64(pTask->taskId);
...@@ -1066,12 +1098,12 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, ...@@ -1066,12 +1098,12 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
} }
SResReadyReq *pMsg = msg; SResReadyReq *pMsg = msg;
pMsg->header.vgId = htonl(addr->nodeId); pMsg->header.vgId = htonl(addr->nodeId);
pMsg->sId = htobe64(schMgmt.sId); pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(pJob->queryId); pMsg->queryId = htobe64(pJob->queryId);
pMsg->taskId = htobe64(pTask->taskId); pMsg->taskId = htobe64(pTask->taskId);
break; break;
} }
case TDMT_VND_FETCH: { case TDMT_VND_FETCH: {
...@@ -1081,31 +1113,31 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, ...@@ -1081,31 +1113,31 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_TASK_ELOG("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SResFetchReq *pMsg = msg; SResFetchReq *pMsg = msg;
pMsg->header.vgId = htonl(addr->nodeId); pMsg->header.vgId = htonl(addr->nodeId);
pMsg->sId = htobe64(schMgmt.sId); pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(pJob->queryId); pMsg->queryId = htobe64(pJob->queryId);
pMsg->taskId = htobe64(pTask->taskId); pMsg->taskId = htobe64(pTask->taskId);
break; break;
} }
case TDMT_VND_DROP_TASK:{ case TDMT_VND_DROP_TASK: {
msgSize = sizeof(STaskDropReq); msgSize = sizeof(STaskDropReq);
msg = calloc(1, msgSize); msg = calloc(1, msgSize);
if (NULL == msg) { if (NULL == msg) {
SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_TASK_ELOG("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
STaskDropReq *pMsg = msg; STaskDropReq *pMsg = msg;
pMsg->header.vgId = htonl(addr->nodeId); pMsg->header.vgId = htonl(addr->nodeId);
pMsg->sId = htobe64(schMgmt.sId); pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(pJob->queryId); pMsg->queryId = htobe64(pJob->queryId);
pMsg->taskId = htobe64(pTask->taskId); pMsg->taskId = htobe64(pTask->taskId);
break; break;
} }
default: default:
...@@ -1121,7 +1153,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, ...@@ -1121,7 +1153,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
if (isCandidateAddr) { if (isCandidateAddr) {
SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr));
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
...@@ -1138,25 +1170,24 @@ static FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { ...@@ -1138,25 +1170,24 @@ static FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) {
*pStatus = status; *pStatus = status;
} }
return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED ||
|| status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING); status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING);
} }
// Note: no more error processing, handled in function internal // Note: no more error processing, handled in function internal
int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) {
int8_t status = 0; int8_t status = 0;
int32_t code = 0; int32_t code = 0;
if (schJobNeedToStop(pJob, &status)) { if (schJobNeedToStop(pJob, &status)) {
SCH_TASK_ELOG("no need to launch task cause of job status, job status:%d", status); SCH_TASK_ELOG("no need to launch task cause of job status, job status:%d", status);
code = atomic_load_32(&pJob->errCode); code = atomic_load_32(&pJob->errCode);
SCH_ERR_RET(code); SCH_ERR_RET(code);
SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
SSubplan *plan = pTask->plan; SSubplan *plan = pTask->plan;
if (NULL == pTask->msg) { if (NULL == pTask->msg) {
...@@ -1168,7 +1199,7 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { ...@@ -1168,7 +1199,7 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) {
printf("physical plan:%s\n", pTask->msg); printf("physical plan:%s\n", pTask->msg);
} }
SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask));
// NOTE: race condition: the task should be put into the hash table before send msg to server // NOTE: race condition: the task should be put into the hash table before send msg to server
...@@ -1179,13 +1210,13 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { ...@@ -1179,13 +1210,13 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) {
} }
SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code)); SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code));
SCH_RET(code); SCH_RET(code);
} }
...@@ -1193,12 +1224,12 @@ int32_t schLaunchJob(SSchJob *pJob) { ...@@ -1193,12 +1224,12 @@ int32_t schLaunchJob(SSchJob *pJob) {
SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx);
SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_EXECUTING)); SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_EXECUTING));
for (int32_t i = 0; i < level->taskNum; ++i) { for (int32_t i = 0; i < level->taskNum; ++i) {
SSchTask *pTask = taosArrayGet(level->subTasks, i); SSchTask *pTask = taosArrayGet(level->subTasks, i);
SCH_ERR_RET(schLaunchTask(pJob, pTask)); SCH_ERR_RET(schLaunchTask(pJob, pTask));
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -1209,7 +1240,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { ...@@ -1209,7 +1240,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) {
} }
int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs);
if (size <= 0) { if (size <= 0) {
SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask));
return; return;
...@@ -1233,9 +1264,9 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { ...@@ -1233,9 +1264,9 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) {
if (!SCH_TASK_NO_NEED_DROP(pTask)) { if (!SCH_TASK_NO_NEED_DROP(pTask)) {
schDropTaskOnExecutedNode(pJob, pTask); schDropTaskOnExecutedNode(pJob, pTask);
} }
pIter = taosHashIterate(list, pIter); pIter = taosHashIterate(list, pIter);
} }
} }
void schDropJobAllTasks(SSchJob *pJob) { void schDropJobAllTasks(SSchJob *pJob) {
...@@ -1244,15 +1275,15 @@ void schDropJobAllTasks(SSchJob *pJob) { ...@@ -1244,15 +1275,15 @@ void schDropJobAllTasks(SSchJob *pJob) {
schDropTaskInHashList(pJob, pJob->failTasks); schDropTaskInHashList(pJob, pJob->failTasks);
} }
int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag *pDag, struct SSchJob **job, bool syncSchedule) {
if (nodeList && taosArrayGetSize(nodeList) <= 0) { if (nodeList && taosArrayGetSize(nodeList) <= 0) {
qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); qInfo("QID:%" PRIx64 " input nodeList is empty", pDag->queryId);
} }
int32_t code = 0; int32_t code = 0;
SSchJob *pJob = calloc(1, sizeof(SSchJob)); SSchJob *pJob = calloc(1, sizeof(SSchJob));
if (NULL == pJob) { if (NULL == pJob) {
qError("QID:%"PRIx64" calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob));
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
...@@ -1262,19 +1293,22 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struc ...@@ -1262,19 +1293,22 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struc
SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob)); SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob));
pJob->execTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); pJob->execTasks =
taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
if (NULL == pJob->execTasks) { if (NULL == pJob->execTasks) {
SCH_JOB_ELOG("taosHashInit %d execTasks failed", pDag->numOfSubplans); SCH_JOB_ELOG("taosHashInit %d execTasks failed", pDag->numOfSubplans);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
pJob->succTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); pJob->succTasks =
taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
if (NULL == pJob->succTasks) { if (NULL == pJob->succTasks) {
SCH_JOB_ELOG("taosHashInit %d succTasks failed", pDag->numOfSubplans); SCH_JOB_ELOG("taosHashInit %d succTasks failed", pDag->numOfSubplans);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
pJob->failTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); pJob->failTasks =
taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
if (NULL == pJob->failTasks) { if (NULL == pJob->failTasks) {
SCH_JOB_ELOG("taosHashInit %d failTasks failed", pDag->numOfSubplans); SCH_JOB_ELOG("taosHashInit %d failTasks failed", pDag->numOfSubplans);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
...@@ -1294,11 +1328,11 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struc ...@@ -1294,11 +1328,11 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struc
} }
pJob->status = JOB_TASK_STATUS_NOT_START; pJob->status = JOB_TASK_STATUS_NOT_START;
SCH_ERR_JRET(schLaunchJob(pJob)); SCH_ERR_JRET(schLaunchJob(pJob));
*(SSchJob **)job = pJob; *(SSchJob **)job = pJob;
if (syncSchedule) { if (syncSchedule) {
SCH_JOB_DLOG("will wait for rsp now, job status:%d", SCH_GET_JOB_STATUS(pJob)); SCH_JOB_DLOG("will wait for rsp now, job status:%d", SCH_GET_JOB_STATUS(pJob));
tsem_wait(&pJob->rspSem); tsem_wait(&pJob->rspSem);
...@@ -1311,20 +1345,18 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struc ...@@ -1311,20 +1345,18 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struc
_return: _return:
*(SSchJob **)job = NULL; *(SSchJob **)job = NULL;
scheduleFreeJob(pJob); scheduleFreeJob(pJob);
SCH_RET(code); SCH_RET(code);
} }
int32_t schCancelJob(SSchJob *pJob) { int32_t schCancelJob(SSchJob *pJob) {
//TODO // TODO
//TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST
// TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST
} }
int32_t schedulerInit(SSchedulerCfg *cfg) { int32_t schedulerInit(SSchedulerCfg *cfg) {
if (schMgmt.jobs) { if (schMgmt.jobs) {
qError("scheduler already initialized"); qError("scheduler already initialized");
...@@ -1333,7 +1365,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { ...@@ -1333,7 +1365,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
if (cfg) { if (cfg) {
schMgmt.cfg = *cfg; schMgmt.cfg = *cfg;
if (schMgmt.cfg.maxJobNum == 0) { if (schMgmt.cfg.maxJobNum == 0) {
schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER;
} }
...@@ -1341,7 +1373,8 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { ...@@ -1341,7 +1373,8 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER;
} }
schMgmt.jobs = taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); schMgmt.jobs =
taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
if (NULL == schMgmt.jobs) { if (NULL == schMgmt.jobs) {
qError("init schduler jobs failed, num:%u", schMgmt.cfg.maxJobNum); qError("init schduler jobs failed, num:%u", schMgmt.cfg.maxJobNum);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
...@@ -1352,12 +1385,12 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { ...@@ -1352,12 +1385,12 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR);
} }
qInfo("scheduler %"PRIx64" initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); qInfo("scheduler %" PRIx64 " initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag *pDag, struct SSchJob **pJob, SQueryResult *pRes) {
if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
...@@ -1370,11 +1403,11 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru ...@@ -1370,11 +1403,11 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru
pRes->code = atomic_load_32(&job->errCode); pRes->code = atomic_load_32(&job->errCode);
pRes->numOfRows = job->resNumOfRows; pRes->numOfRows = job->resNumOfRows;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag *pDag, struct SSchJob **pJob) {
if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
...@@ -1383,7 +1416,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, ...@@ -1383,7 +1416,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { int32_t schedulerConvertDagToTaskList(SQueryDag *pDag, SArray **pTasks) {
if (NULL == pDag || pDag->numOfSubplans <= 0 || taosArrayGetSize(pDag->pSubplans) <= 0) { if (NULL == pDag || pDag->numOfSubplans <= 0 || taosArrayGetSize(pDag->pSubplans) <= 0) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
...@@ -1408,10 +1441,10 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { ...@@ -1408,10 +1441,10 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) {
} }
STaskInfo tInfo = {0}; STaskInfo tInfo = {0};
char *msg = NULL; char * msg = NULL;
int32_t msgLen = 0; int32_t msgLen = 0;
int32_t code = 0; int32_t code = 0;
for (int32_t i = 0; i < taskNum; ++i) { for (int32_t i = 0; i < taskNum; ++i) {
SSubplan *plan = taosArrayGetP(plans, i); SSubplan *plan = taosArrayGetP(plans, i);
...@@ -1429,14 +1462,14 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { ...@@ -1429,14 +1462,14 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) {
qError("calloc %d failed", msgSize); qError("calloc %d failed", msgSize);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SSubQueryMsg *pMsg = msg; SSubQueryMsg *pMsg = msg;
pMsg->header.vgId = htonl(tInfo.addr.nodeId); pMsg->header.vgId = htonl(tInfo.addr.nodeId);
pMsg->sId = htobe64(schMgmt.sId); pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(plan->id.queryId); pMsg->queryId = htobe64(plan->id.queryId);
pMsg->taskId = htobe64(atomic_add_fetch_64(&schMgmt.taskId, 1)); pMsg->taskId = htobe64(schGenUUID());
pMsg->contentLen = htonl(msgLen); pMsg->contentLen = htonl(msgLen);
memcpy(pMsg->msg, msg, msgLen); memcpy(pMsg->msg, msg, msgLen);
...@@ -1451,7 +1484,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { ...@@ -1451,7 +1484,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) {
*pTasks = info; *pTasks = info;
info = NULL; info = NULL;
_return: _return:
schedulerFreeTaskList(info); schedulerFreeTaskList(info);
...@@ -1459,40 +1492,81 @@ _return: ...@@ -1459,40 +1492,81 @@ _return:
SCH_RET(code); SCH_RET(code);
} }
int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) {
if (NULL == src || NULL == dst || copyNum <= 0) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
int32_t code = 0;
*dst = taosArrayInit(copyNum, sizeof(STaskInfo));
if (NULL == *dst) {
qError("taosArrayInit %d taskInfo failed", copyNum);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
int32_t msgSize = src->msg->contentLen + sizeof(*src->msg);
STaskInfo info = {0};
info.addr = src->addr;
for (int32_t i = 0; i < copyNum; ++i) {
info.msg = malloc(msgSize);
if (NULL == info.msg) {
qError("malloc %d failed", msgSize);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
memcpy(info.msg, src->msg, msgSize);
info.msg->taskId = schGenUUID();
if (NULL == taosArrayPush(*dst, &info)) {
qError("taosArrayPush failed, idx:%d", i);
free(info.msg);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
}
return TSDB_CODE_SUCCESS;
int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { _return:
schedulerFreeTaskList(*dst);
*dst = NULL;
SCH_RET(code);
}
int32_t scheduleFetchRows(SSchJob *pJob, void **pData) {
if (NULL == pJob || NULL == pData) { if (NULL == pJob || NULL == pData) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
int32_t code = 0; int32_t code = 0;
atomic_add_fetch_32(&pJob->ref, 1);
int8_t status = SCH_GET_JOB_STATUS(pJob); int8_t status = SCH_GET_JOB_STATUS(pJob);
if (status == JOB_TASK_STATUS_DROPPING) { if (status == JOB_TASK_STATUS_DROPPING) {
SCH_JOB_ELOG("job is dropping, status:%d", status); SCH_JOB_ELOG("job is dropping, status:%d", status);
return TSDB_CODE_SCH_STATUS_ERROR; SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR);
} }
atomic_add_fetch_32(&pJob->ref, 1);
if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { if (!SCH_JOB_NEED_FETCH(&pJob->attr)) {
SCH_JOB_ELOG("no need to fetch data, status:%d", SCH_GET_JOB_STATUS(pJob)); SCH_JOB_ELOG("no need to fetch data, status:%d", SCH_GET_JOB_STATUS(pJob));
atomic_sub_fetch_32(&pJob->ref, 1); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
if (atomic_val_compare_exchange_8(&pJob->userFetch, 0, 1) != 0) { if (atomic_val_compare_exchange_8(&pJob->userFetch, 0, 1) != 0) {
SCH_JOB_ELOG("prior fetching not finished, userFetch:%d", atomic_load_8(&pJob->userFetch)); SCH_JOB_ELOG("prior fetching not finished, userFetch:%d", atomic_load_8(&pJob->userFetch));
atomic_sub_fetch_32(&pJob->ref, 1); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
if (status == JOB_TASK_STATUS_FAILED) { if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) {
*pData = atomic_load_ptr(&pJob->res); SCH_JOB_ELOG("job failed or dropping, status:%d", status);
atomic_store_ptr(&pJob->res, NULL);
SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); SCH_ERR_JRET(atomic_load_32(&pJob->errCode));
} else if (status == JOB_TASK_STATUS_SUCCEED) { } else if (status == JOB_TASK_STATUS_SUCCEED) {
*pData = atomic_load_ptr(&pJob->res); SCH_JOB_ELOG("job already succeed, status:%d", status);
atomic_store_ptr(&pJob->res, NULL);
goto _return; goto _return;
} else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) {
SCH_ERR_JRET(schFetchFromRemote(pJob)); SCH_ERR_JRET(schFetchFromRemote(pJob));
...@@ -1502,18 +1576,20 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { ...@@ -1502,18 +1576,20 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) {
status = SCH_GET_JOB_STATUS(pJob); status = SCH_GET_JOB_STATUS(pJob);
if (status == JOB_TASK_STATUS_FAILED) { if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) {
code = atomic_load_32(&pJob->errCode); SCH_JOB_ELOG("job failed or dropping, status:%d", status);
SCH_ERR_JRET(code); SCH_ERR_JRET(atomic_load_32(&pJob->errCode));
} }
if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) {
SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED));
} }
_return:
while (true) { while (true) {
*pData = atomic_load_ptr(&pJob->res); *pData = atomic_load_ptr(&pJob->res);
if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) { if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) {
continue; continue;
} }
...@@ -1521,10 +1597,19 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { ...@@ -1521,10 +1597,19 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) {
break; break;
} }
_return: if (NULL == *pData) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp));
if (rsp) {
rsp->completed = 1;
}
*pData = rsp;
}
atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
SCH_JOB_DLOG("fetch done, code:%x", code);
atomic_sub_fetch_32(&pJob->ref, 1); atomic_sub_fetch_32(&pJob->ref, 1);
SCH_RET(code); SCH_RET(code);
...@@ -1549,6 +1634,7 @@ void scheduleFreeJob(void *job) { ...@@ -1549,6 +1634,7 @@ void scheduleFreeJob(void *job) {
SSchJob *pJob = job; SSchJob *pJob = job;
uint64_t queryId = pJob->queryId; uint64_t queryId = pJob->queryId;
bool setJobFree = false;
if (SCH_GET_JOB_STATUS(pJob) > 0) { if (SCH_GET_JOB_STATUS(pJob) > 0) {
if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) {
...@@ -1556,8 +1642,6 @@ void scheduleFreeJob(void *job) { ...@@ -1556,8 +1642,6 @@ void scheduleFreeJob(void *job) {
return; return;
} }
schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING);
SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref));
while (true) { while (true) {
...@@ -1565,6 +1649,11 @@ void scheduleFreeJob(void *job) { ...@@ -1565,6 +1649,11 @@ void scheduleFreeJob(void *job) {
if (0 == ref) { if (0 == ref) {
break; break;
} else if (ref > 0) { } else if (ref > 0) {
if (1 == ref && atomic_load_8(&pJob->userFetch) > 0 && !setJobFree) {
schProcessOnJobDropped(pJob, TSDB_CODE_QRY_JOB_FREED);
setJobFree = true;
}
usleep(1); usleep(1);
} else { } else {
assert(0); assert(0);
...@@ -1580,32 +1669,33 @@ void scheduleFreeJob(void *job) { ...@@ -1580,32 +1669,33 @@ void scheduleFreeJob(void *job) {
schDropJobAllTasks(pJob); schDropJobAllTasks(pJob);
} }
pJob->subPlans = NULL; // it is a reference to pDag->pSubplans pJob->subPlans = NULL; // it is a reference to pDag->pSubplans
int32_t numOfLevels = taosArrayGetSize(pJob->levels); int32_t numOfLevels = taosArrayGetSize(pJob->levels);
for(int32_t i = 0; i < numOfLevels; ++i) { for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i); SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for(int32_t j = 0; j < numOfTasks; ++j) { for (int32_t j = 0; j < numOfTasks; ++j) {
SSchTask* pTask = taosArrayGet(pLevel->subTasks, j); SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
schFreeTask(pTask); schFreeTask(pTask);
} }
taosArrayDestroy(pLevel->subTasks); taosArrayDestroy(pLevel->subTasks);
} }
taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->execTasks);
taosHashCleanup(pJob->failTasks); taosHashCleanup(pJob->failTasks);
taosHashCleanup(pJob->succTasks); taosHashCleanup(pJob->succTasks);
taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->levels);
taosArrayDestroy(pJob->nodeList);
tfree(pJob->res); tfree(pJob->res);
tfree(pJob); tfree(pJob);
qDebug("QID:%"PRIx64" job freed", queryId); qDebug("QID:%" PRIx64 " job freed", queryId);
} }
void schedulerFreeTaskList(SArray *taskList) { void schedulerFreeTaskList(SArray *taskList) {
...@@ -1621,11 +1711,10 @@ void schedulerFreeTaskList(SArray *taskList) { ...@@ -1621,11 +1711,10 @@ void schedulerFreeTaskList(SArray *taskList) {
taosArrayDestroy(taskList); taosArrayDestroy(taskList);
} }
void schedulerDestroy(void) { void schedulerDestroy(void) {
if (schMgmt.jobs) { if (schMgmt.jobs) {
taosHashCleanup(schMgmt.jobs); //TODO taosHashCleanup(schMgmt.jobs); // TODO
schMgmt.jobs = NULL; schMgmt.jobs = NULL;
} }
} }
source/libs/scheduler/test/schedulerTests.cpp
浏览文件 @ f9d7cc44
...@@ -38,6 +38,20 @@ ...@@ -38,6 +38,20 @@
namespace { namespace {
extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode);
extern "C" int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode);
struct SSchJob *pInsertJob = NULL;
struct SSchJob *pQueryJob = NULL;
uint64_t schtMergeTemplateId = 0x4;
uint64_t schtFetchTaskId = 0;
uint64_t schtQueryId = 1;
bool schtTestStop = false;
bool schtTestDeadLoop = false;
int32_t schtTestMTRunSec = 10;
int32_t schtTestPrintNum = 1000;
int32_t schtStartFetch = 0;
void schtInitLogFile() { void schtInitLogFile() {
...@@ -57,7 +71,7 @@ void schtInitLogFile() { ...@@ -57,7 +71,7 @@ void schtInitLogFile() {
void schtBuildQueryDag(SQueryDag *dag) { void schtBuildQueryDag(SQueryDag *dag) {
uint64_t qId = 0x0000000000000001; uint64_t qId = schtQueryId;
dag->queryId = qId; dag->queryId = qId;
dag->numOfSubplans = 2; dag->numOfSubplans = 2;
...@@ -84,7 +98,7 @@ void schtBuildQueryDag(SQueryDag *dag) { ...@@ -84,7 +98,7 @@ void schtBuildQueryDag(SQueryDag *dag) {
scanPlan->msgType = TDMT_VND_QUERY; scanPlan->msgType = TDMT_VND_QUERY;
mergePlan->id.queryId = qId; mergePlan->id.queryId = qId;
mergePlan->id.templateId = 0x4444444444; mergePlan->id.templateId = schtMergeTemplateId;
mergePlan->id.subplanId = 0x5555555555; mergePlan->id.subplanId = 0x5555555555;
mergePlan->type = QUERY_TYPE_MERGE; mergePlan->type = QUERY_TYPE_MERGE;
mergePlan->level = 0; mergePlan->level = 0;
...@@ -173,8 +187,6 @@ void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int6 ...@@ -173,8 +187,6 @@ void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int6
} }
void schtSetPlanToString() { void schtSetPlanToString() {
static Stub stub; static Stub stub;
stub.set(qSubPlanToString, schtPlanToString); stub.set(qSubPlanToString, schtPlanToString);
...@@ -214,7 +226,12 @@ void schtSetRpcSendRequest() { ...@@ -214,7 +226,12 @@ void schtSetRpcSendRequest() {
} }
} }
int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) {
if (pInfo) {
tfree(pInfo->param);
tfree(pInfo->msgInfo.pData);
free(pInfo);
}
return 0; return 0;
} }
...@@ -269,15 +286,224 @@ void *schtCreateFetchRspThread(void *param) { ...@@ -269,15 +286,224 @@ void *schtCreateFetchRspThread(void *param) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp));
rsp->completed = 1; rsp->completed = 1;
rsp->numOfRows = 10; rsp->numOfRows = 10;
code = schHandleResponseMsg(job, job->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(rsp), 0);
code = schHandleResponseMsg(job, job->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0);
assert(code == 0); assert(code == 0);
} }
void *schtFetchRspThread(void *aa) {
SDataBuf dataBuf = {0};
SSchCallbackParam* param = NULL;
while (!schtTestStop) {
if (0 == atomic_val_compare_exchange_32(&schtStartFetch, 1, 0)) {
continue;
}
usleep(1);
param = (SSchCallbackParam *)calloc(1, sizeof(*param));
param->queryId = schtQueryId;
param->taskId = schtFetchTaskId;
int32_t code = 0;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp));
rsp->completed = 1;
rsp->numOfRows = 10;
dataBuf.pData = rsp;
dataBuf.len = sizeof(*rsp);
code = schHandleCallback(param, &dataBuf, TDMT_VND_FETCH_RSP, 0);
assert(code == 0 || code);
}
}
void schtFreeQueryJob(int32_t freeThread) {
static uint32_t freeNum = 0;
SSchJob *job = atomic_load_ptr(&pQueryJob);
if (job && atomic_val_compare_exchange_ptr(&pQueryJob, job, NULL)) {
scheduleFreeJob(job);
if (freeThread) {
if (++freeNum % schtTestPrintNum == 0) {
printf("FreeNum:%d\n", freeNum);
}
}
}
}
void* schtRunJobThread(void *aa) {
void *mockPointer = (void *)0x1;
char *clusterId = "cluster1";
char *dbname = "1.db1";
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
SQueryDag dag = {0};
schtInitLogFile();
int32_t code = schedulerInit(NULL);
assert(code == 0);
schtSetPlanToString();
schtSetExecNode();
schtSetAsyncSendMsgToServer();
SSchJob *job = NULL;
SSchCallbackParam *param = NULL;
SHashObj *execTasks = NULL;
SDataBuf dataBuf = {0};
uint32_t jobFinished = 0;
while (!schtTestStop) {
schtBuildQueryDag(&dag);
SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr));
SEpAddr qnodeAddr = {0};
strcpy(qnodeAddr.fqdn, "qnode0.ep");
qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr);
code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &job);
assert(code == 0);
execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
void *pIter = taosHashIterate(job->execTasks, NULL);
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
schtFetchTaskId = task->taskId - 1;
taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task));
pIter = taosHashIterate(job->execTasks, pIter);
}
param = (SSchCallbackParam *)calloc(1, sizeof(*param));
param->queryId = schtQueryId;
pQueryJob = job;
pIter = taosHashIterate(execTasks, NULL);
while (pIter) {
SSchTask *task = (SSchTask *)pIter;
param->taskId = task->taskId;
SQueryTableRsp rsp = {0};
dataBuf.pData = &rsp;
dataBuf.len = sizeof(rsp);
code = schHandleCallback(param, &dataBuf, TDMT_VND_QUERY_RSP, 0);
assert(code == 0 || code);
pIter = taosHashIterate(execTasks, pIter);
}
param = (SSchCallbackParam *)calloc(1, sizeof(*param));
param->queryId = schtQueryId;
pIter = taosHashIterate(execTasks, NULL);
while (pIter) {
SSchTask *task = (SSchTask *)pIter;
param->taskId = task->taskId;
SResReadyRsp rsp = {0};
dataBuf.pData = &rsp;
dataBuf.len = sizeof(rsp);
code = schHandleCallback(param, &dataBuf, TDMT_VND_RES_READY_RSP, 0);
assert(code == 0 || code);
pIter = taosHashIterate(execTasks, pIter);
}
param = (SSchCallbackParam *)calloc(1, sizeof(*param));
param->queryId = schtQueryId;
pIter = taosHashIterate(execTasks, NULL);
while (pIter) {
SSchTask *task = (SSchTask *)pIter;
param->taskId = task->taskId - 1;
SQueryTableRsp rsp = {0};
dataBuf.pData = &rsp;
dataBuf.len = sizeof(rsp);
code = schHandleCallback(param, &dataBuf, TDMT_VND_QUERY_RSP, 0);
assert(code == 0 || code);
pIter = taosHashIterate(execTasks, pIter);
}
param = (SSchCallbackParam *)calloc(1, sizeof(*param));
param->queryId = schtQueryId;
pIter = taosHashIterate(execTasks, NULL);
while (pIter) {
SSchTask *task = (SSchTask *)pIter;
param->taskId = task->taskId - 1;
SResReadyRsp rsp = {0};
dataBuf.pData = &rsp;
dataBuf.len = sizeof(rsp);
code = schHandleCallback(param, &dataBuf, TDMT_VND_RES_READY_RSP, 0);
assert(code == 0 || code);
pIter = taosHashIterate(execTasks, pIter);
}
atomic_store_32(&schtStartFetch, 1);
void *data = NULL;
code = scheduleFetchRows(pQueryJob, &data);
assert(code == 0 || code);
if (0 == code) {
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
assert(pRsp->completed == 1);
assert(pRsp->numOfRows == 10);
}
data = NULL;
code = scheduleFetchRows(pQueryJob, &data);
assert(code == 0 || code);
schtFreeQueryJob(0);
taosHashCleanup(execTasks);
schtFreeQueryDag(&dag);
if (++jobFinished % schtTestPrintNum == 0) {
printf("jobFinished:%d\n", jobFinished);
}
++schtQueryId;
}
schedulerDestroy();
}
void* schtFreeJobThread(void *aa) {
while (!schtTestStop) {
usleep(rand() % 100);
schtFreeQueryJob(1);
}
}
struct SSchJob *pInsertJob = NULL;
} }
TEST(queryTest, normalCase) { TEST(queryTest, normalCase) {
...@@ -368,11 +594,12 @@ TEST(queryTest, normalCase) { ...@@ -368,11 +594,12 @@ TEST(queryTest, normalCase) {
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
ASSERT_EQ(pRsp->completed, 1); ASSERT_EQ(pRsp->completed, 1);
ASSERT_EQ(pRsp->numOfRows, 10); ASSERT_EQ(pRsp->numOfRows, 10);
tfree(data);
data = NULL; data = NULL;
code = scheduleFetchRows(job, &data); code = scheduleFetchRows(job, &data);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(data, (void*)NULL); ASSERT_TRUE(data);
scheduleFreeJob(pJob); scheduleFreeJob(pJob);
...@@ -383,7 +610,6 @@ TEST(queryTest, normalCase) { ...@@ -383,7 +610,6 @@ TEST(queryTest, normalCase) {
TEST(insertTest, normalCase) { TEST(insertTest, normalCase) {
void *mockPointer = (void *)0x1; void *mockPointer = (void *)0x1;
char *clusterId = "cluster1"; char *clusterId = "cluster1";
...@@ -427,13 +653,29 @@ TEST(insertTest, normalCase) { ...@@ -427,13 +653,29 @@ TEST(insertTest, normalCase) {
} }
TEST(multiThread, forceFree) { TEST(multiThread, forceFree) {
pthread_attr_t thattr;
pthread_attr_init(&thattr);
schtInitLogFile(); pthread_t thread1, thread2, thread3;
pthread_create(&(thread1), &thattr, schtRunJobThread, NULL);
pthread_create(&(thread2), &thattr, schtFreeJobThread, NULL);
pthread_create(&(thread3), &thattr, schtFetchRspThread, NULL);
while (true) {
if (schtTestDeadLoop) {
sleep(1);
} else {
sleep(schtTestMTRunSec);
break;
}
}
schtTestStop = true;
sleep(3);
} }
int main(int argc, char** argv) { int main(int argc, char** argv) {
srand(time(NULL));
testing::InitGoogleTest(&argc, argv); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
} }
......
source/libs/tfs/inc/tfsInt.h
浏览文件 @ f9d7cc44
...@@ -33,47 +33,69 @@ extern int32_t fsDebugFlag; ...@@ -33,47 +33,69 @@ extern int32_t fsDebugFlag;
#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }} #define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }}
#define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }} #define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }}
#define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", 255, __VA_ARGS__); }} #define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", 255, __VA_ARGS__); }}
#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }} #define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }}
#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }} #define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }}
// Global Definitions typedef struct {
#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
typedef struct SDisk {
int32_t level; int32_t level;
int32_t id; int32_t id;
char *path; char *path;
SDiskSize size; SDiskSize size;
} SDisk; } STfsDisk;
typedef struct STier { typedef struct {
pthread_spinlock_t lock; pthread_spinlock_t lock;
int32_t level; int32_t level;
int16_t nextid; // next disk id to allocate int32_t nextid; // next disk id to allocate
int16_t ndisk; // # of disks mounted to this tier int32_t ndisk; // # of disks mounted to this tier
int16_t nAvailDisks; // # of Available disks int32_t nAvailDisks; // # of Available disks
SDisk *disks[TSDB_MAX_DISKS_PER_TIER]; STfsDisk *disks[TFS_MAX_DISKS_PER_TIER];
SDiskSize size;
} STfsTier;
typedef struct {
STfsDisk *pDisk;
} SDiskIter;
typedef struct STfsDir {
SDiskIter iter;
SDiskID did;
char dirname[TSDB_FILENAME_LEN];
STfsFile tfile;
DIR *dir;
STfs *pTfs;
} STfsDir;
typedef struct STfs {
pthread_spinlock_t lock;
SDiskSize size; SDiskSize size;
} STier; int32_t nlevel;
STfsTier tiers[TFS_MAX_TIERS];
#define TIER_LEVEL(pt) ((pt)->level) SHashObj *hash; // name to did map
#define TIER_NDISKS(pt) ((pt)->ndisk) } STfs;
#define TIER_SIZE(pt) ((pt)->tmeta.size)
#define TIER_FREE_SIZE(pt) ((pt)->tmeta.free) STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir);
STfsDisk *tfsFreeDisk(STfsDisk *pDisk);
#define DISK_AT_TIER(pt, id) ((pt)->disks[id]) int32_t tfsUpdateDiskSize(STfsDisk *pDisk);
#define DISK_DIR(pd) ((pd)->path)
int32_t tfsInitTier(STfsTier *pTier, int32_t level);
SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir); void tfsDestroyTier(STfsTier *pTier);
SDisk *tfsFreeDisk(SDisk *pDisk); STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg);
int32_t tfsUpdateDiskSize(SDisk *pDisk); void tfsUpdateTierSize(STfsTier *pTier);
int32_t tfsAllocDiskOnTier(STfsTier *pTier);
int32_t tfsInitTier(STier *pTier, int32_t level); void tfsPosNextId(STfsTier *pTier);
void tfsDestroyTier(STier *pTier);
SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg); #define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock)
void tfsUpdateTierSize(STier *pTier); #define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock)
int32_t tfsAllocDiskOnTier(STier *pTier);
void tfsPosNextId(STier *pTier); #define tfsLock(pTfs) pthread_spin_lock(&(pTfs)->lock)
#define tfsUnLock(pTfs) pthread_spin_unlock(&(pTfs)->lock)
#define TFS_TIER_AT(pTfs, level) (&(pTfs)->tiers[level])
#define TFS_DISK_AT(pTfs, did) ((pTfs)->tiers[(did).level].disks[(did).id])
#define TFS_PRIMARY_DISK(pTfs) ((pTfs)->tiers[0].disks[0])
#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/libs/tfs/src/tfs.c
浏览文件 @ f9d7cc44
...@@ -16,227 +16,200 @@ ...@@ -16,227 +16,200 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tfsInt.h" #include "tfsInt.h"
#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg);
static int32_t tfsCheck(STfs *pTfs);
typedef struct { static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg);
pthread_spinlock_t lock; static int32_t tfsFormatDir(char *idir, char *odir);
SFSMeta meta; static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir);
int32_t nlevel; static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir);
STier tiers[TSDB_MAX_TIERS]; static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter);
SHashObj *map; // name to did map
} SFS; STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) {
if (ndisk < 0 || ndisk > TFS_MAX_DISKS) {
typedef struct {
SDisk *pDisk;
} SDiskIter;
#define TFS_META() (pfs->meta)
#define TFS_NLEVEL() (pfs->nlevel)
#define TFS_TIERS() (pfs->tiers)
#define TFS_TIER_AT(level) (TFS_TIERS() + (level))
#define TFS_DISK_AT(level, id) DISK_AT_TIER(TFS_TIER_AT(level), id)
#define TFS_PRIMARY_DISK() TFS_DISK_AT(TFS_PRIMARY_LEVEL, TFS_PRIMARY_ID)
#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL()))
#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level))))
#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id))
#define tfsLock() pthread_spin_lock(&(pfs->lock))
#define tfsUnLock() pthread_spin_unlock(&(pfs->lock))
static SFS tfs = {0};
static SFS *pfs = &tfs;
// STATIC DECLARATION
static int32_t tfsMount(SDiskCfg *pCfg);
static int32_t tfsCheck();
static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg);
static int32_t tfsFormatDir(char *idir, char *odir);
static SDisk *tfsGetDiskByID(SDiskID did);
static SDisk *tfsGetDiskByName(const char *dir);
static int32_t tfsOpendirImpl(TDIR *tdir);
static void tfsInitDiskIter(SDiskIter *pIter);
static SDisk *tfsNextDisk(SDiskIter *pIter);
// FS APIs ====================================
int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk) {
if (ndisk < 0) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return -1; return NULL;
} }
for (int32_t level = 0; level < TSDB_MAX_TIERS; level++) { STfs *pTfs = calloc(1, sizeof(STfs));
if (tfsInitTier(TFS_TIER_AT(level), level) < 0) { if (pTfs == NULL) {
while (true) { terrno = TSDB_CODE_OUT_OF_MEMORY;
level--; return NULL;
if (level < 0) break; }
tfsDestroyTier(TFS_TIER_AT(level)); if (pthread_spin_init(&pTfs->lock, 0) != 0) {
} terrno = TAOS_SYSTEM_ERROR(errno);
tfsClose(pTfs);
return NULL;
}
return -1; for (int32_t level = 0; level < TFS_MAX_TIERS; level++) {
STfsTier *pTier = &pTfs->tiers[level];
if (tfsInitTier(pTier, level) < 0) {
tfsClose(pTfs);
return NULL;
} }
} }
pthread_spin_init(&(pfs->lock), 0); pTfs->hash = taosHashInit(TFS_MAX_DISKS * 2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pTfs->hash == NULL) {
pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pfs->map == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
tfsCleanup(); tfsClose(pTfs);
return -1; return NULL;
} }
for (int32_t idisk = 0; idisk < ndisk; idisk++) { for (int32_t idisk = 0; idisk < ndisk; idisk++) {
if (tfsMount(pDiskCfg + idisk) < 0) { if (tfsMount(pTfs, &pCfg[idisk]) < 0) {
tfsCleanup(); tfsClose(pTfs);
return -1; return NULL;
} }
} }
if (tfsCheck() < 0) { if (tfsCheck(pTfs) < 0) {
tfsCleanup(); tfsClose(pTfs);
return -1; return NULL;
} }
tfsUpdateSize(NULL); tfsUpdateSize(pTfs);
for (int32_t level = 0; level < TFS_NLEVEL(); level++) { for (int32_t level = 0; level < pTfs->nlevel; level++) {
tfsPosNextId(TFS_TIER_AT(level)); tfsPosNextId(&pTfs->tiers[level]);
} }
return 0; return pTfs;
} }
void tfsCleanup() { void tfsClose(STfs *pTfs) {
taosHashCleanup(pfs->map); if (pTfs == NULL) return;
pfs->map = NULL;
pthread_spin_destroy(&(pfs->lock)); for (int32_t level = 0; level < TFS_MAX_LEVEL; level++) {
for (int32_t level = 0; level < TFS_NLEVEL(); level++) { tfsDestroyTier(&pTfs->tiers[level]);
tfsDestroyTier(TFS_TIER_AT(level));
} }
taosHashCleanup(pTfs->hash);
pthread_spin_destroy(&pTfs->lock);
free(pTfs);
} }
void tfsUpdateSize(SFSMeta *pFSMeta) { void tfsUpdateSize(STfs *pTfs) {
SFSMeta fsMeta = {0};
SDiskSize size = {0}; SDiskSize size = {0};
if (pFSMeta == NULL) { for (int32_t level = 0; level < pTfs->nlevel; level++) {
pFSMeta = &fsMeta; STfsTier *pTier = &pTfs->tiers[level];
}
memset(pFSMeta, 0, sizeof(SFSMeta));
for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
STier *pTier = TFS_TIER_AT(level);
tfsUpdateTierSize(pTier); tfsUpdateTierSize(pTier);
pFSMeta->total += pTier->size.total; size.total += pTier->size.total;
pFSMeta->avail += pTier->size.avail; size.avail += pTier->size.avail;
pFSMeta->used += pTier->size.used; size.used += pTier->size.used;
} }
tfsLock(); tfsLock(pTfs);
pfs->meta = *pFSMeta; pTfs->size = size;
tfsUnLock(); tfsUnLock(pTfs);
} }
/* Allocate an existing available tier level SDiskSize tfsGetSize(STfs *pTfs) {
*/ tfsLock(pTfs);
void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id) { SDiskSize size = pTfs->size;
ASSERT(expLevel >= 0); tfsUnLock(pTfs);
return size;
}
*level = expLevel; int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) {
*id = TFS_UNDECIDED_ID; pDiskId->level = expLevel;
pDiskId->id = -1;
if (*level >= TFS_NLEVEL()) { if (pDiskId->level >= pTfs->nlevel) {
*level = TFS_NLEVEL() - 1; pDiskId->level--;
} }
while (*level >= 0) { while (pDiskId->level >= 0) {
*id = tfsAllocDiskOnTier(TFS_TIER_AT(*level)); pDiskId->id = tfsAllocDiskOnTier(&pTfs->tiers[pDiskId->level]);
if (*id == TFS_UNDECIDED_ID) { if (pDiskId->id < 0) {
(*level)--; pDiskId->level--;
continue; continue;
} }
return; return 0;
} }
*level = TFS_UNDECIDED_LEVEL; terrno = TSDB_CODE_FS_NO_VALID_DISK;
*id = TFS_UNDECIDED_ID; return -1;
} }
const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); } const char *tfsGetPrimaryPath(STfs *pTfs) { return TFS_PRIMARY_DISK(pTfs)->path; }
const char *TFS_DISK_PATH(int32_t level, int32_t id) { return DISK_DIR(TFS_DISK_AT(level, id)); }
// TFILE APIs ==================================== const char *tfsGetDiskPath(STfs *pTfs, SDiskID diskId) { return TFS_DISK_AT(pTfs, diskId)->path; }
void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname) {
ASSERT(TFS_IS_VALID_DISK(level, id));
SDisk *pDisk = TFS_DISK_AT(level, id); void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname) {
STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
pf->level = level; pFile->did = diskId;
pf->id = id; tstrncpy(pFile->rname, rname, TSDB_FILENAME_LEN);
tstrncpy(pf->rname, bname, TSDB_FILENAME_LEN);
char tmpName[TMPNAME_LEN] = {0}; char tmpName[TMPNAME_LEN] = {0};
snprintf(tmpName, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), bname); snprintf(tmpName, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
tstrncpy(pf->aname, tmpName, TSDB_FILENAME_LEN); tstrncpy(pFile->aname, tmpName, TSDB_FILENAME_LEN);
pFile->pTfs = pTfs;
} }
bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) { bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) {
ASSERT(pf1 != NULL || pf2 != NULL); if (pFile1 == NULL || pFile2 == NULL || pFile1->pTfs != pFile2->pTfs) return false;
if (pf1 == NULL || pf2 == NULL) return false; if (pFile1->did.level != pFile2->did.level) return false;
if (pf1->level != pf2->level) return false; if (pFile1->did.id != pFile2->did.id) return false;
if (pf1->id != pf2->id) return false; if (strncmp(pFile1->rname, pFile2->rname, TSDB_FILENAME_LEN) != 0) return false;
if (strncmp(pf1->rname, pf2->rname, TSDB_FILENAME_LEN) != 0) return false;
return true; return true;
} }
int32_t tfsEncodeFile(void **buf, TFILE *pf) { int32_t tfsEncodeFile(void **buf, STfsFile *pFile) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeVariantI32(buf, pf->level); tlen += taosEncodeVariantI32(buf, pFile->did.level);
tlen += taosEncodeVariantI32(buf, pf->id); tlen += taosEncodeVariantI32(buf, pFile->did.id);
tlen += taosEncodeString(buf, pf->rname); tlen += taosEncodeString(buf, pFile->rname);
return tlen; return tlen;
} }
void *tfsDecodeFile(void *buf, TFILE *pf) { void *tfsDecodeFile(STfs *pTfs, void *buf, STfsFile *pFile) {
int32_t level, id; SDiskID diskId = {0};
char *rname; char *rname = NULL;
buf = taosDecodeVariantI32(buf, &(level)); buf = taosDecodeVariantI32(buf, &diskId.level);
buf = taosDecodeVariantI32(buf, &(id)); buf = taosDecodeVariantI32(buf, &diskId.id);
buf = taosDecodeString(buf, &rname); buf = taosDecodeString(buf, &rname);
tfsInitFile(pf, level, id, rname); tfsInitFile(pTfs, pFile, diskId, rname);
tfree(rname); tfree(rname);
return buf; return buf;
} }
void tfsbasename(const TFILE *pf, char *dest) { void tfsBasename(const STfsFile *pFile, char *dest) {
char tname[TSDB_FILENAME_LEN] = "\0"; char tname[TSDB_FILENAME_LEN] = "\0";
tstrncpy(tname, pf->aname, TSDB_FILENAME_LEN); tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN);
tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN); tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN);
} }
void tfsdirname(const TFILE *pf, char *dest) { void tfsDirname(const STfsFile *pFile, char *dest) {
char tname[TSDB_FILENAME_LEN] = "\0"; char tname[TSDB_FILENAME_LEN] = "\0";
tstrncpy(tname, pf->aname, TSDB_FILENAME_LEN); tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN);
tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN);
} }
// DIR APIs ==================================== int32_t tfsRemoveFile(const STfsFile *pFile) {
int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id) { return remove(pFile->aname);
SDisk *pDisk = TFS_DISK_AT(level, id); }
char aname[TMPNAME_LEN];
int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) {
return taosCopyFile(pFile1->aname, pFile2->aname);
}
snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId) {
STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
char aname[TMPNAME_LEN];
snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
if (taosMkDir(aname) != 0) { if (taosMkDir(aname) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
...@@ -245,8 +218,8 @@ int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id) { ...@@ -245,8 +218,8 @@ int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id) {
return 0; return 0;
} }
int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) {
if (tfsMkdirAt(rname, level, id) < 0) { if (tfsMkdirAt(pTfs, rname, diskId) < 0) {
if (errno == ENOENT) { if (errno == ENOENT) {
// Try to create upper // Try to create upper
char *s = strdup(rname); char *s = strdup(rname);
...@@ -259,7 +232,7 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { ...@@ -259,7 +232,7 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) {
// https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html
char *dir = strdup(dirname(s)); char *dir = strdup(dirname(s));
if (tfsMkdirRecurAt(dir, level, id) < 0) { if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) {
free(s); free(s);
free(dir); free(dir);
return -1; return -1;
...@@ -267,7 +240,7 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { ...@@ -267,7 +240,7 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) {
free(s); free(s);
free(dir); free(dir);
if (tfsMkdirAt(rname, level, id) < 0) { if (tfsMkdirAt(pTfs, rname, diskId) < 0) {
return -1; return -1;
} }
} else { } else {
...@@ -278,11 +251,12 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { ...@@ -278,11 +251,12 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) {
return 0; return 0;
} }
int32_t tfsMkdir(const char *rname) { int32_t tfsMkdir(STfs *pTfs, const char *rname) {
for (int32_t level = 0; level < TFS_NLEVEL(); level++) { for (int32_t level = 0; level < pTfs->nlevel; level++) {
STier *pTier = TFS_TIER_AT(level); STfsTier *pTier = TFS_TIER_AT(pTfs, level);
for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { for (int32_t id = 0; id < pTier->ndisk; id++) {
if (tfsMkdirAt(rname, level, id) < 0) { SDiskID did = {.id = id, .level = level};
if (tfsMkdirAt(pTfs, rname, did) < 0) {
return -1; return -1;
} }
} }
...@@ -291,16 +265,14 @@ int32_t tfsMkdir(const char *rname) { ...@@ -291,16 +265,14 @@ int32_t tfsMkdir(const char *rname) {
return 0; return 0;
} }
int32_t tfsRmdir(const char *rname) { int32_t tfsRmdir(STfs *pTfs, const char *rname) {
char aname[TMPNAME_LEN] = "\0"; char aname[TMPNAME_LEN] = "\0";
for (int32_t level = 0; level < TFS_NLEVEL(); level++) { for (int32_t level = 0; level < pTfs->nlevel; level++) {
STier *pTier = TFS_TIER_AT(level); STfsTier *pTier = TFS_TIER_AT(pTfs, level);
for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { for (int32_t id = 0; id < pTier->ndisk; id++) {
SDisk *pDisk = pTier->disks[id]; STfsDisk *pDisk = pTier->disks[id];
snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
snprintf(aname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TS_PATH_DELIMITER, rname);
taosRemoveDir(aname); taosRemoveDir(aname);
} }
} }
...@@ -308,117 +280,108 @@ int32_t tfsRmdir(const char *rname) { ...@@ -308,117 +280,108 @@ int32_t tfsRmdir(const char *rname) {
return 0; return 0;
} }
#if 0 int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) {
int32_t tfsRename(char *orname, char *nrname) {
char oaname[TMPNAME_LEN] = "\0"; char oaname[TMPNAME_LEN] = "\0";
char naname[TMPNAME_LEN] = "\0"; char naname[TMPNAME_LEN] = "\0";
for (int32_t level = 0; level < pfs->nlevel; level++) { for (int32_t level = 0; level < pTfs->nlevel; level++) {
STier *pTier = TFS_TIER_AT(level); STfsTier *pTier = TFS_TIER_AT(pTfs, level);
for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { for (int32_t id = 0; id < pTier->ndisk; id++) {
SDisk *pDisk = DISK_AT_TIER(pTier, id); STfsDisk *pDisk = pTier->disks[id];
snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname);
snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname); snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname);
snprintf(naname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), nrname); if (taosRenameFile(oaname, naname) != 0) {
return -1;
taosRenameFile(oaname, naname); }
} }
} }
return 0; return 0;
} }
#endif
STfsDir *tfsOpendir(STfs *pTfs, const char *rname) {
struct TDIR { STfsDir *pDir = calloc(1, sizeof(STfsDir));
SDiskIter iter; if (pDir == NULL) {
int32_t level;
int32_t id;
char dirname[TSDB_FILENAME_LEN];
TFILE tfile;
DIR *dir;
};
TDIR *tfsOpendir(const char *rname) {
TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir));
if (tdir == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
tfsInitDiskIter(&(tdir->iter)); SDiskID diskId = {.id = 0, .level = 0};
tstrncpy(tdir->dirname, rname, TSDB_FILENAME_LEN); pDir->iter.pDisk = TFS_DISK_AT(pTfs, diskId);
pDir->pTfs = pTfs;
tstrncpy(pDir->dirname, rname, TSDB_FILENAME_LEN);
if (tfsOpendirImpl(tdir) < 0) { if (tfsOpendirImpl(pTfs, pDir) < 0) {
free(tdir); free(pDir);
return NULL; return NULL;
} }
return tdir; return pDir;
} }
const TFILE *tfsReaddir(TDIR *tdir) { const STfsFile *tfsReaddir(STfsDir *pDir) {
if (tdir == NULL || tdir->dir == NULL) return NULL; if (pDir == NULL || pDir->dir == NULL) return NULL;
char bname[TMPNAME_LEN * 2] = "\0"; char bname[TMPNAME_LEN * 2] = "\0";
while (true) { while (true) {
struct dirent *dp = NULL; struct dirent *dp = NULL;
dp = readdir(tdir->dir); dp = readdir(pDir->dir);
if (dp != NULL) { if (dp != NULL) {
// Skip . and .. // Skip . and ..
if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue;
snprintf(bname, TMPNAME_LEN * 2, "%s/%s", tdir->dirname, dp->d_name); snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name);
tfsInitFile(&(tdir->tfile), tdir->level, tdir->id, bname); tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname);
return &(tdir->tfile); return &pDir->tfile;
} }
if (tfsOpendirImpl(tdir) < 0) { if (tfsOpendirImpl(pDir->pTfs, pDir) < 0) {
return NULL; return NULL;
} }
if (tdir->dir == NULL) { if (pDir->dir == NULL) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
return NULL; return NULL;
} }
} }
} }
void tfsClosedir(TDIR *tdir) { void tfsClosedir(STfsDir *pDir) {
if (tdir) { if (pDir) {
if (tdir->dir != NULL) { if (pDir->dir != NULL) {
closedir(tdir->dir); closedir(pDir->dir);
tdir->dir = NULL; pDir->dir = NULL;
} }
free(tdir); free(pDir);
} }
} }
// private static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) {
static int32_t tfsMount(SDiskCfg *pCfg) { if (tfsCheckAndFormatCfg(pTfs, pCfg) < 0) {
SDiskID did; return -1;
SDisk *pDisk = NULL; }
if (tfsCheckAndFormatCfg(pCfg) < 0) return -1;
did.level = pCfg->level; SDiskID did = {.level = pCfg->level};
pDisk = tfsMountDiskToTier(TFS_TIER_AT(did.level), pCfg); STfsDisk *pDisk = tfsMountDiskToTier(TFS_TIER_AT(pTfs, did.level), pCfg);
if (pDisk == NULL) { if (pDisk == NULL) {
fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, terrstr());
return -1; return -1;
} }
did.id = pDisk->id; did.id = pDisk->id;
taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did)); taosHashPut(pTfs->hash, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did));
if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1; if (pTfs->nlevel < pCfg->level + 1) {
pTfs->nlevel = pCfg->level + 1;
}
return 0; return 0;
} }
static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) { static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) {
char dirName[TSDB_FILENAME_LEN] = "\0"; char dirName[TSDB_FILENAME_LEN] = "\0";
struct stat pstat; struct stat pstat;
if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) { if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) {
fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level);
terrno = TSDB_CODE_FS_INVLD_CFG; terrno = TSDB_CODE_FS_INVLD_CFG;
return -1; return -1;
...@@ -431,7 +394,7 @@ static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) { ...@@ -431,7 +394,7 @@ static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
return -1; return -1;
} }
if (TFS_PRIMARY_DISK() != NULL) { if (TFS_PRIMARY_DISK(pTfs) != NULL) {
fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir); fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir);
terrno = TSDB_CODE_FS_DUP_PRIMARY; terrno = TSDB_CODE_FS_DUP_PRIMARY;
return -1; return -1;
...@@ -444,7 +407,7 @@ static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) { ...@@ -444,7 +407,7 @@ static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
return -1; return -1;
} }
if (tfsGetDiskByName(dirName) != NULL) { if (tfsGetDiskByName(pTfs, dirName) != NULL) {
fError("failed to mount %s to FS since duplicate mount", pCfg->dir); fError("failed to mount %s to FS since duplicate mount", pCfg->dir);
terrno = TSDB_CODE_FS_INVLD_CFG; terrno = TSDB_CODE_FS_INVLD_CFG;
return -1; return -1;
...@@ -494,15 +457,15 @@ static int32_t tfsFormatDir(char *idir, char *odir) { ...@@ -494,15 +457,15 @@ static int32_t tfsFormatDir(char *idir, char *odir) {
return 0; return 0;
} }
static int32_t tfsCheck() { static int32_t tfsCheck(STfs *pTfs) {
if (TFS_PRIMARY_DISK() == NULL) { if (TFS_PRIMARY_DISK(pTfs) == NULL) {
fError("no primary disk is set"); fError("no primary disk is set");
terrno = TSDB_CODE_FS_NO_PRIMARY_DISK; terrno = TSDB_CODE_FS_NO_PRIMARY_DISK;
return -1; return -1;
} }
for (int32_t level = 0; level < TFS_NLEVEL(); level++) { for (int32_t level = 0; level < pTfs->nlevel; level++) {
if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) { if (TFS_TIER_AT(pTfs, level)->ndisk == 0) {
fError("no disk at level %d", level); fError("no disk at level %d", level);
terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER; terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER;
return -1; return -1;
...@@ -512,66 +475,55 @@ static int32_t tfsCheck() { ...@@ -512,66 +475,55 @@ static int32_t tfsCheck() {
return 0; return 0;
} }
static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); } static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir) {
static SDisk *tfsGetDiskByName(const char *dir) { void *pr = taosHashGet(pTfs->hash, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN));
SDiskID did;
SDisk *pDisk = NULL;
void *pr = NULL;
pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN));
if (pr == NULL) return NULL; if (pr == NULL) return NULL;
did = *(SDiskID *)pr; SDiskID did = *(SDiskID *)pr;
pDisk = tfsGetDiskByID(did); STfsDisk *pDisk = TFS_DISK_AT(pTfs, did);
ASSERT(pDisk != NULL);
return pDisk; return pDisk;
} }
static int32_t tfsOpendirImpl(TDIR *tdir) { static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) {
SDisk *pDisk = NULL; STfsDisk *pDisk = NULL;
char adir[TMPNAME_LEN * 2] = "\0"; char adir[TMPNAME_LEN * 2] = "\0";
if (tdir->dir != NULL) { if (pDir->dir != NULL) {
closedir(tdir->dir); closedir(pDir->dir);
tdir->dir = NULL; pDir->dir = NULL;
} }
while (true) { while (true) {
pDisk = tfsNextDisk(&(tdir->iter)); pDisk = tfsNextDisk(pTfs, &pDir->iter);
if (pDisk == NULL) return 0; if (pDisk == NULL) return 0;
tdir->level = pDisk->level; pDir->did.level = pDisk->level;
tdir->id = pDisk->id; pDir->did.id = pDisk->id;
snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TS_PATH_DELIMITER,tdir->dirname); snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname);
tdir->dir = opendir(adir); pDir->dir = opendir(adir);
if (tdir->dir != NULL) break; if (pDir->dir != NULL) break;
} }
return 0; return 0;
} }
static void tfsInitDiskIter(SDiskIter *pIter) { pIter->pDisk = TFS_DISK_AT(0, 0); } static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter) {
if (pIter == NULL) return NULL;
static SDisk *tfsNextDisk(SDiskIter *pIter) {
SDisk *pDisk = pIter->pDisk;
STfsDisk *pDisk = pIter->pDisk;
if (pDisk == NULL) return NULL; if (pDisk == NULL) return NULL;
int32_t level = pDisk->level; SDiskID did = {.level = pDisk->level, .id = pDisk->id + 1};
int32_t id = pDisk->id;
id++; if (did.id < TFS_TIER_AT(pTfs, did.level)->ndisk) {
if (id < TIER_NDISKS(TFS_TIER_AT(level))) { pIter->pDisk = TFS_DISK_AT(pTfs, did);
pIter->pDisk = TFS_DISK_AT(level, id);
ASSERT(pIter->pDisk != NULL);
} else { } else {
level++; did.level++;
id = 0; did.id = 0;
if (level < TFS_NLEVEL()) { if (did.level < pTfs->nlevel) {
pIter->pDisk = TFS_DISK_AT(level, id); pIter->pDisk = TFS_DISK_AT(pTfs, did);
ASSERT(pIter->pDisk != NULL);
} else { } else {
pIter->pDisk = NULL; pIter->pDisk = NULL;
} }
...@@ -579,25 +531,3 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) { ...@@ -579,25 +531,3 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) {
return pDisk; return pDisk;
} }
// OTHER FUNCTIONS ===================================
void taosGetDisk() {
const double unit = 1024 * 1024 * 1024;
SDiskSize diskSize;
SFSMeta fsMeta;
tfsUpdateSize(&fsMeta);
tsTotalDataDirGB = (float)(fsMeta.total / unit);
tsUsedDataDirGB = (float)(fsMeta.used / unit);
tsAvailDataDirGB = (float)(fsMeta.avail / unit);
if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
tsTotalLogDirGB = (float)(diskSize.total / unit);
tsAvailLogDirGB = (float)(diskSize.avail / unit);
}
if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
tsTotalTmpDirGB = (float)(diskSize.total / unit);
tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
}
}
source/libs/tfs/src/tfsDisk.c
浏览文件 @ f9d7cc44
...@@ -16,8 +16,8 @@ ...@@ -16,8 +16,8 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tfsInt.h" #include "tfsInt.h"
SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) {
SDisk *pDisk = calloc(1, sizeof(SDisk)); STfsDisk *pDisk = calloc(1, sizeof(STfsDisk));
if (pDisk == NULL) { if (pDisk == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
...@@ -36,7 +36,7 @@ SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { ...@@ -36,7 +36,7 @@ SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) {
return pDisk; return pDisk;
} }
SDisk *tfsFreeDisk(SDisk *pDisk) { STfsDisk *tfsFreeDisk(STfsDisk *pDisk) {
if (pDisk != NULL) { if (pDisk != NULL) {
free(pDisk->path); free(pDisk->path);
free(pDisk); free(pDisk);
...@@ -45,8 +45,8 @@ SDisk *tfsFreeDisk(SDisk *pDisk) { ...@@ -45,8 +45,8 @@ SDisk *tfsFreeDisk(SDisk *pDisk) {
return NULL; return NULL;
} }
int32_t tfsUpdateDiskSize(SDisk *pDisk) { int32_t tfsUpdateDiskSize(STfsDisk *pDisk) {
if (taosGetDiskSize(pDisk->path, &pDisk->size) != 0) { if (taosGetDiskSize(pDisk->path, &pDisk->size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr()); fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr());
return -1; return -1;
......
source/libs/tfs/src/tfsTier.c
浏览文件 @ f9d7cc44
...@@ -16,11 +16,8 @@ ...@@ -16,11 +16,8 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tfsInt.h" #include "tfsInt.h"
#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock) int32_t tfsInitTier(STfsTier *pTier, int32_t level) {
#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) memset(pTier, 0, sizeof(STfsTier));
int32_t tfsInitTier(STier *pTier, int32_t level) {
memset(pTier, 0, sizeof(STier));
if (pthread_spin_init(&pTier->lock, 0) != 0) { if (pthread_spin_init(&pTier->lock, 0) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
...@@ -31,17 +28,17 @@ int32_t tfsInitTier(STier *pTier, int32_t level) { ...@@ -31,17 +28,17 @@ int32_t tfsInitTier(STier *pTier, int32_t level) {
return 0; return 0;
} }
void tfsDestroyTier(STier *pTier) { void tfsDestroyTier(STfsTier *pTier) {
for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) { for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; id++) {
pTier->disks[id] = tfsFreeDisk(pTier->disks[id]); pTier->disks[id] = tfsFreeDisk(pTier->disks[id]);
} }
pTier->ndisk = 0; pTier->ndisk = 0;
pthread_spin_destroy(&(pTier->lock)); pthread_spin_destroy(&pTier->lock);
} }
SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) {
if (pTier->ndisk >= TSDB_MAX_DISKS_PER_TIER) { if (pTier->ndisk >= TFS_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
return NULL; return NULL;
} }
...@@ -61,12 +58,12 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { ...@@ -61,12 +58,12 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
id = pTier->ndisk; id = pTier->ndisk;
} }
if (id >= TSDB_MAX_DISKS_PER_TIER) { if (id >= TFS_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
return NULL; return NULL;
} }
SDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); STfsDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
if (pDisk == NULL) return NULL; if (pDisk == NULL) return NULL;
pTier->disks[id] = pDisk; pTier->disks[id] = pDisk;
...@@ -76,15 +73,16 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { ...@@ -76,15 +73,16 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
return pTier->disks[id]; return pTier->disks[id];
} }
void tfsUpdateTierSize(STier *pTier) { void tfsUpdateTierSize(STfsTier *pTier) {
SDiskSize size = {0}; SDiskSize size = {0};
int16_t nAvailDisks = 0; int32_t nAvailDisks = 0;
tfsLockTier(pTier); tfsLockTier(pTier);
for (int32_t id = 0; id < pTier->ndisk; id++) { for (int32_t id = 0; id < pTier->ndisk; id++) {
SDisk *pDisk = pTier->disks[id]; STfsDisk *pDisk = pTier->disks[id];
if (pDisk == NULL) continue; if (pDisk == NULL) continue;
if (tfsUpdateDiskSize(pDisk) < 0) continue;
size.total += pDisk->size.total; size.total += pDisk->size.total;
size.used += pDisk->size.used; size.used += pDisk->size.used;
...@@ -99,7 +97,7 @@ void tfsUpdateTierSize(STier *pTier) { ...@@ -99,7 +97,7 @@ void tfsUpdateTierSize(STier *pTier) {
} }
// Round-Robin to allocate disk on a tier // Round-Robin to allocate disk on a tier
int32_t tfsAllocDiskOnTier(STier *pTier) { int32_t tfsAllocDiskOnTier(STfsTier *pTier) {
terrno = TSDB_CODE_FS_NO_VALID_DISK; terrno = TSDB_CODE_FS_NO_VALID_DISK;
tfsLockTier(pTier); tfsLockTier(pTier);
...@@ -110,9 +108,9 @@ int32_t tfsAllocDiskOnTier(STier *pTier) { ...@@ -110,9 +108,9 @@ int32_t tfsAllocDiskOnTier(STier *pTier) {
} }
int32_t retId = -1; int32_t retId = -1;
for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; ++id) { for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; ++id) {
int32_t diskId = (pTier->nextid + id) % pTier->ndisk; int32_t diskId = (pTier->nextid + id) % pTier->ndisk;
SDisk *pDisk = pTier->disks[diskId]; STfsDisk *pDisk = pTier->disks[diskId];
if (pDisk == NULL) continue; if (pDisk == NULL) continue;
...@@ -128,12 +126,12 @@ int32_t tfsAllocDiskOnTier(STier *pTier) { ...@@ -128,12 +126,12 @@ int32_t tfsAllocDiskOnTier(STier *pTier) {
return retId; return retId;
} }
void tfsPosNextId(STier *pTier) { void tfsPosNextId(STfsTier *pTier) {
int32_t nextid = 0; int32_t nextid = 0;
for (int32_t id = 1; id < pTier->ndisk; id++) { for (int32_t id = 1; id < pTier->ndisk; id++) {
SDisk *pLDisk = pTier->disks[nextid]; STfsDisk *pLDisk = pTier->disks[nextid];
SDisk *pDisk = pTier->disks[id]; STfsDisk *pDisk = pTier->disks[id];
if (pDisk->size.avail > TFS_MIN_DISK_FREE_SIZE && pDisk->size.avail > pLDisk->size.avail) { if (pDisk->size.avail > TFS_MIN_DISK_FREE_SIZE && pDisk->size.avail > pLDisk->size.avail) {
nextid = id; nextid = id;
} }
......
source/libs/transport/inc/transComm.h
浏览文件 @ f9d7cc44
...@@ -68,6 +68,25 @@ typedef void* queue[2]; ...@@ -68,6 +68,25 @@ typedef void* queue[2];
QUEUE_PREV_NEXT(e) = QUEUE_NEXT(e); \ QUEUE_PREV_NEXT(e) = QUEUE_NEXT(e); \
QUEUE_NEXT_PREV(e) = QUEUE_PREV(e); \ QUEUE_NEXT_PREV(e) = QUEUE_PREV(e); \
} }
#define QUEUE_SPLIT(h, q, n) \
do { \
QUEUE_PREV(n) = QUEUE_PREV(h); \
QUEUE_PREV_NEXT(n) = (n); \
QUEUE_NEXT(n) = (q); \
QUEUE_PREV(h) = QUEUE_PREV(q); \
QUEUE_PREV_NEXT(h) = (h); \
QUEUE_PREV(q) = (n); \
} while (0)
#define QUEUE_MOVE(h, n) \
do { \
if (QUEUE_IS_EMPTY(h)) { \
QUEUE_INIT(n); \
} else { \
queue* q = QUEUE_HEAD(h); \
QUEUE_SPLIT(h, q, n); \
} \
} while (0)
/* Return the element at the front of the queue. */ /* Return the element at the front of the queue. */
#define QUEUE_HEAD(q) (QUEUE_NEXT(q)) #define QUEUE_HEAD(q) (QUEUE_NEXT(q))
......
source/libs/transport/src/trans.c
浏览文件 @ f9d7cc44
...@@ -35,6 +35,7 @@ void* rpcOpen(const SRpcInit* pInit) { ...@@ -35,6 +35,7 @@ void* rpcOpen(const SRpcInit* pInit) {
if (pInit->label) { if (pInit->label) {
tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); tstrncpy(pRpc->label, pInit->label, strlen(pInit->label));
} }
pRpc->cfp = pInit->cfp;
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
pRpc->connType = pInit->connType; pRpc->connType = pInit->connType;
pRpc->tcphandle = (*taosHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); pRpc->tcphandle = (*taosHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc);
......
source/libs/transport/src/transCli.c
浏览文件 @ f9d7cc44
...@@ -20,12 +20,16 @@ ...@@ -20,12 +20,16 @@
typedef struct SCliConn { typedef struct SCliConn {
uv_connect_t connReq; uv_connect_t connReq;
uv_stream_t* stream; uv_stream_t* stream;
uv_write_t* writeReq;
void* data; void* data;
queue conn; queue conn;
char spi;
char secured;
} SCliConn; } SCliConn;
typedef struct SCliMsg { typedef struct SCliMsg {
SRpcReqContext* context; SRpcReqContext* context;
queue q; queue q;
uint64_t st;
} SCliMsg; } SCliMsg;
typedef struct SCliThrdObj { typedef struct SCliThrdObj {
...@@ -45,86 +49,169 @@ typedef struct SClientObj { ...@@ -45,86 +49,169 @@ typedef struct SClientObj {
SCliThrdObj** pThreadObj; SCliThrdObj** pThreadObj;
} SClientObj; } SClientObj;
static void clientWriteCb(uv_write_t* req, int status); // conn pool
static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port);
static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn);
static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
static void clientConnCb(struct uv_connect_s* req, int status); static void clientWriteCb(uv_write_t* req, int status);
static void clientConnCb(uv_connect_t* req, int status);
static void clientAsyncCb(uv_async_t* handle); static void clientAsyncCb(uv_async_t* handle);
static void clientDestroy(uv_handle_t* handle);
static void clientConnDestroy(SCliConn* pConn);
static void* clientThread(void* arg); static void* clientThread(void* arg);
static void clientWriteCb(uv_write_t* req, int status) { static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
// impl later // impl later
} }
static void clientFailedCb(uv_handle_t* handle) { static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
// impl later // impl later
tDebug("close handle"); SCliConn* conn = handle->data;
if (nread > 0) {
return;
}
//
uv_close((uv_handle_t*)handle, clientDestroy);
} }
static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
static void clientConnDestroy(SCliConn* conn) {
// impl later // impl later
//
}
static void clientDestroy(uv_handle_t* handle) {
SCliConn* conn = handle->data;
clientConnDestroy(conn);
} }
static void clientConnCb(struct uv_connect_s* req, int status) {
static void clientWriteCb(uv_write_t* req, int status) {
SCliConn* pConn = req->data; SCliConn* pConn = req->data;
if (status == 0) {
tDebug("data already was written on stream");
} else {
uv_close((uv_handle_t*)pConn->stream, clientDestroy);
return;
}
uv_read_start((uv_stream_t*)pConn->stream, clientAllocrReadBufferCb, clientReadCb);
// impl later
}
static void clientWrite(SCliConn* pConn) {
SCliMsg* pMsg = pConn->data; SCliMsg* pMsg = pConn->data;
SEpSet* pEpSet = &pMsg->context->epSet; SRpcHead* pHead = rpcHeadFromCont(pMsg->context->pCont);
int msgLen = rpcMsgLenFromCont(pMsg->context->contLen);
char* msg = (char*)(pHead);
uv_buf_t wb = uv_buf_init(msg, msgLen);
uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb);
}
static void clientConnCb(uv_connect_t* req, int status) {
// impl later
SCliConn* pConn = req->data;
if (status != 0) {
tError("failed to connect %s", uv_err_name(status));
clientConnDestroy(pConn);
return;
}
SCliMsg* pMsg = pConn->data;
SEpSet* pEpSet = &pMsg->context->epSet;
SRpcMsg rpcMsg;
// rpcMsg.ahandle = pMsg->context->ahandle;
// rpcMsg.pCont = NULL;
char* fqdn = pEpSet->fqdn[pEpSet->inUse]; char* fqdn = pEpSet->fqdn[pEpSet->inUse];
uint32_t port = pEpSet->port[pEpSet->inUse]; uint32_t port = pEpSet->port[pEpSet->inUse];
if (status != 0) { if (status != 0) {
// call user fp later // call user fp later
tError("failed to connect server(%s, %d), errmsg: %s", fqdn, port, uv_strerror(status)); tError("failed to connect server(%s, %d), errmsg: %s", fqdn, port, uv_strerror(status));
uv_close((uv_handle_t*)req->handle, clientFailedCb); SRpcInfo* pRpc = pMsg->context->pRpc;
(pRpc->cfp)(NULL, &rpcMsg, pEpSet);
uv_close((uv_handle_t*)req->handle, clientDestroy);
return; return;
} }
assert(pConn->stream == req->handle); assert(pConn->stream == req->handle);
// impl later
} }
static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) {
// impl later // impl later
return NULL; return NULL;
} }
static void clientAsyncCb(uv_async_t* handle) { static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) {
SCliThrdObj* pThrd = handle->data; // impl later
SCliMsg* pMsg = NULL; }
pthread_mutex_lock(&pThrd->msgMtx);
if (!QUEUE_IS_EMPTY(&pThrd->msg)) { static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
queue* head = QUEUE_HEAD(&pThrd->msg); SEpSet* pEpSet = &pMsg->context->epSet;
pMsg = QUEUE_DATA(head, SCliMsg, q);
QUEUE_REMOVE(head);
}
pthread_mutex_unlock(&pThrd->msgMtx);
SEpSet* pEpSet = &pMsg->context->epSet;
char* fqdn = pEpSet->fqdn[pEpSet->inUse]; char* fqdn = pEpSet->fqdn[pEpSet->inUse];
uint32_t port = pEpSet->port[pEpSet->inUse]; uint32_t port = pEpSet->port[pEpSet->inUse];
uint64_t el = taosGetTimestampUs() - pMsg->st;
tDebug("msg tran time cost: %" PRIu64 "", el);
SCliConn* conn = getConnFromCache(pThrd->cache, fqdn, port); SCliConn* conn = getConnFromCache(pThrd->cache, fqdn, port);
if (conn != NULL) { if (conn != NULL) {
// impl later // impl later
conn->data = pMsg;
conn->writeReq->data = conn;
clientWrite(conn);
// uv_buf_t wb;
// uv_write(conn->writeReq, (uv_stream_t*)conn->stream, &wb, 1, clientWriteCb);
} else { } else {
SCliConn* conn = malloc(sizeof(SCliConn)); SCliConn* conn = malloc(sizeof(SCliConn));
conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t));
uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream));
conn->writeReq = malloc(sizeof(uv_write_t));
conn->connReq.data = conn; conn->connReq.data = conn;
conn->data = pMsg; conn->data = pMsg;
struct sockaddr_in addr; struct sockaddr_in addr;
uv_ip4_addr(fqdn, port, &addr); uv_ip4_addr(fqdn, port, &addr);
// handle error in callback if connect error // handle error in callback if fail to connect
uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb);
}
// SRpcReqContext* pCxt = pMsg->context; // SRpcMsg rpcMsg;
// SEpSet* pEpSet = &pMsg->context->epSet;
// SRpcInfo* pRpc = pMsg->context->pRpc;
//// rpcMsg.ahandle = pMsg->context->ahandle;
// rpcMsg.pCont = NULL;
// rpcMsg.ahandle = pMsg->context->ahandle;
// uint64_t el1 = taosGetTimestampUs() - et;
// tError("msg tran back first: time cost: %" PRIu64 "", el1);
// et = taosGetTimestampUs();
//(pRpc->cfp)(NULL, &rpcMsg, pEpSet);
// uint64_t el2 = taosGetTimestampUs() - et;
// tError("msg tran back second: time cost: %" PRIu64 "", el2);
}
}
static void clientAsyncCb(uv_async_t* handle) {
SCliThrdObj* pThrd = handle->data;
SCliMsg* pMsg = NULL;
queue wq;
// SRpcHead* pHead = rpcHeadFromCont(pCtx->pCont); // batch process to avoid to lock/unlock frequently
// char* msg = (char*)pHead; pthread_mutex_lock(&pThrd->msgMtx);
// int len = rpcMsgLenFromCont(pCtx->contLen); QUEUE_MOVE(&pThrd->msg, &wq);
// tmsg_t msgType = pCtx->msgType; pthread_mutex_unlock(&pThrd->msgMtx);
// impl later int count = 0;
while (!QUEUE_IS_EMPTY(&wq)) {
queue* h = QUEUE_HEAD(&wq);
QUEUE_REMOVE(h);
pMsg = QUEUE_DATA(h, SCliMsg, q);
clientHandleReq(pMsg, pThrd);
count++;
if (count >= 2) {
tError("send batch size: %d", count);
}
}
} }
static void* clientThread(void* arg) { static void* clientThread(void* arg) {
...@@ -142,9 +229,6 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, ...@@ -142,9 +229,6 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj));
QUEUE_INIT(&pThrd->msg); QUEUE_INIT(&pThrd->msg);
pthread_mutex_init(&pThrd->msgMtx, NULL); pthread_mutex_init(&pThrd->msgMtx, NULL);
// QUEUE_INIT(&pThrd->clientCache);
pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
uv_loop_init(pThrd->loop); uv_loop_init(pThrd->loop);
...@@ -186,6 +270,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* ...@@ -186,6 +270,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t*
} }
SCliMsg* msg = malloc(sizeof(SCliMsg)); SCliMsg* msg = malloc(sizeof(SCliMsg));
msg->context = pContext; msg->context = pContext;
msg->st = taosGetTimestampUs();
SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads];
......
source/libs/transport/src/transSrv.c
浏览文件 @ f9d7cc44
...@@ -277,10 +277,6 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { ...@@ -277,10 +277,6 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
} }
return; return;
} }
if (terrno != 0) {
// handle err code
}
if (nread != UV_EOF) { if (nread != UV_EOF) {
tDebug("Read error %s\n", uv_err_name(nread)); tDebug("Read error %s\n", uv_err_name(nread));
} }
...@@ -309,21 +305,23 @@ void uvOnWriteCb(uv_write_t* req, int status) { ...@@ -309,21 +305,23 @@ void uvOnWriteCb(uv_write_t* req, int status) {
void uvWorkerAsyncCb(uv_async_t* handle) { void uvWorkerAsyncCb(uv_async_t* handle) {
SWorkThrdObj* pThrd = container_of(handle, SWorkThrdObj, workerAsync); SWorkThrdObj* pThrd = container_of(handle, SWorkThrdObj, workerAsync);
SConn* conn = NULL; SConn* conn = NULL;
queue wq;
// opt later // batch process to avoid to lock/unlock frequently
pthread_mutex_lock(&pThrd->connMtx); pthread_mutex_lock(&pThrd->connMtx);
if (!QUEUE_IS_EMPTY(&pThrd->conn)) { QUEUE_MOVE(&pThrd->conn, &wq);
queue* head = QUEUE_HEAD(&pThrd->conn);
conn = QUEUE_DATA(head, SConn, queue);
QUEUE_REMOVE(head);
}
pthread_mutex_unlock(&pThrd->connMtx); pthread_mutex_unlock(&pThrd->connMtx);
if (conn == NULL) {
tError("except occurred, do nothing"); while (!QUEUE_IS_EMPTY(&wq)) {
return; queue* head = QUEUE_HEAD(&wq);
QUEUE_REMOVE(head);
SConn* conn = QUEUE_DATA(head, SConn, queue);
if (conn == NULL) {
tError("except occurred, do nothing");
return;
}
uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len);
uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb);
} }
uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len);
uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb);
} }
void uvOnAcceptCb(uv_stream_t* stream, int status) { void uvOnAcceptCb(uv_stream_t* stream, int status) {
......
source/libs/transport/test/rclient.c
浏览文件 @ f9d7cc44
...@@ -34,8 +34,8 @@ typedef struct { ...@@ -34,8 +34,8 @@ typedef struct {
static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
SInfo *pInfo = (SInfo *)pMsg->ahandle; SInfo *pInfo = (SInfo *)pMsg->ahandle;
tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, // tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
pMsg->code); // pMsg->code);
if (pEpSet) pInfo->epSet = *pEpSet; if (pEpSet) pInfo->epSet = *pEpSet;
...@@ -57,7 +57,7 @@ static void *sendRequest(void *param) { ...@@ -57,7 +57,7 @@ static void *sendRequest(void *param) {
rpcMsg.contLen = pInfo->msgSize; rpcMsg.contLen = pInfo->msgSize;
rpcMsg.ahandle = pInfo; rpcMsg.ahandle = pInfo;
rpcMsg.msgType = 1; rpcMsg.msgType = 1;
tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem);
......
source/os/src/osFile.c
浏览文件 @ f9d7cc44
...@@ -142,7 +142,7 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { ...@@ -142,7 +142,7 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) {
int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); }
int64_t taosCopyFile(char *from, char *to) { int64_t taosCopyFile(const char *from, const char *to) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0; return 0;
#else #else
...@@ -400,7 +400,7 @@ int32_t taosFsyncFile(FileFd fd) { ...@@ -400,7 +400,7 @@ int32_t taosFsyncFile(FileFd fd) {
#endif #endif
} }
int32_t taosRenameFile(char *oldName, char *newName) { int32_t taosRenameFile(const char *oldName, const char *newName) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
if (code < 0) { if (code < 0) {
......
source/util/src/terror.c
浏览文件 @ f9d7cc44
...@@ -360,6 +360,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLING, "Task cancelling") ...@@ -360,6 +360,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLING, "Task cancelling")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPING, "Task dropping") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPING, "Task dropping")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUPLICATTED_OPERATION, "Duplicatted operation") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUPLICATTED_OPERATION, "Duplicatted operation")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_MSG_ERROR, "Task message error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_MSG_ERROR, "Task message error")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_FREED, "Job already freed")
......
tests/script/sh/massiveTable/deployCluster.sh
浏览文件 @ f9d7cc44
...@@ -5,21 +5,71 @@ ...@@ -5,21 +5,71 @@
set -e set -e
#set -x #set -x
# deployCluster.sh masterDnode=slave
dataRootDir="/data"
firstEp="trd02:7000"
startPort=7000
dnodeNumber=1
updateSrc=no
while getopts "hm:f:n:r:p:u:" arg
do
case $arg in
m)
masterDnode=$( echo $OPTARG )
;;
n)
dnodeNumber=$(echo $OPTARG)
;;
u)
updateSrc=$(echo $OPTARG)
;;
f)
firstEp=$(echo $OPTARG)
;;
p)
startPort=$(echo $OPTARG)
;;
r)
dataRootDir=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -m [if master dnode] "
echo " -n [ dnode number] "
echo " -f [ first ep] "
echo " -p [ start port] "
echo " -r [ dnode root dir] "
exit 0
;;
?) #unknow option
echo "unkonw argument"
exit 1
;;
esac
done
curr_dir=$(pwd) # deployCluster.sh
echo "currect pwd: ${curr_dir}" curr_dir=$(readlink -f "$(dirname "$0")")
echo $curr_dir
./cleanCluster.sh -r "/data" ${curr_dir}/cleanCluster.sh -r "/data"
./cleanCluster.sh -r "/data2" ${curr_dir}/cleanCluster.sh -r "/data2"
./compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" if [[ "${updateSrc}" == "yes" ]]; then
${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0"
fi
./setupDnodes.sh -r "/data" -n 1 -f "trd02:7000" -p 7000 ${curr_dir}/setupDnodes.sh -r "/data" -n ${dnodeNumber} -f ${firstEp} -p 7000
./setupDnodes.sh -r "/data2" -n 1 -f "trd02:7000" -p 8000 ${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000
#./setupDnodes.sh -r "/data" -n 2 -f trd02:7000 -p 7000 if [[ "${masterDnode}" == "master" ]]; then
#./setupDnodes.sh -r "/data2" -n 2 -f trd02:7000 -p 8000 # create all dnode into cluster
taos -s "create dnode trd02 port 8000;"
taos -s "create dnode trd03 port 7000;"
taos -s "create dnode trd03 port 8000;"
taos -s "create dnode trd04 port 7000;"
taos -s "create dnode trd04 port 8000;"
fi
......
tests/script/sh/massiveTable/setupDnodes.sh
浏览文件 @ f9d7cc44
...@@ -121,7 +121,7 @@ startDnodes() { ...@@ -121,7 +121,7 @@ startDnodes() {
############################### main process ########################################## ############################### main process ##########################################
## kill all taosd process ## kill all taosd process
kill_process taosd #kill_process taosd
## create director for all dnode ## create director for all dnode
if [[ "$enviMode" == "new" ]]; then if [[ "$enviMode" == "new" ]]; then
......
tests/script/sim/table/basic1.sim
浏览文件 @ f9d7cc44
...@@ -67,7 +67,31 @@ sql insert into c1 values(now+1s, 1) ...@@ -67,7 +67,31 @@ sql insert into c1 values(now+1s, 1)
sql insert into c1 values(now+2s, 2) sql insert into c1 values(now+2s, 2)
sql insert into c1 values(now+3s, 3) sql insert into c1 values(now+3s, 3)
return print =============== query data
sql select * from c1
if $rows != 3 then
return -1
endi
print $data00 $data01
print $data10 $data11
print $data20 $data11
if $data01 != 1 then
return -1
endi
if $data11 != 2 then
return -1
endi
if $data21 != 3 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
print =============== query data print =============== query data
sql select * from c1 sql select * from c1
if $rows != 3 then if $rows != 3 then
......
tests/script/tmp/dnodes.sim 已删除 100644 → 0
浏览文件 @ 47c3361e
############## config parameter #####################
$node1 = 192.168.0.201
$node2 = 192.168.0.202
$node3 = 192.168.0.203
$node4 = 192.168.0.204
$self = $node1
$num = 25
#deploy = 0, start = 1, stop = 2
$option = 0
print =============== option:$option
############### stop dnodes #####################
if $option == 0 then
system sh/stop_dnodes.sh
endi
############### process firstEp #####################
$firstEp = $node1 . :7100
$firstPort = 7100
if $self == $node1 then
if $option == 1 then
system sh/exec.sh -n dnode1 -s start
endi
if $option == 2 then
system sh/exec.sh -n dnode1 -s stop -x SIGINT
endi
if $option == 0 then
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp
system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp
system sh/cfg.sh -n dnode1 -c fqdn -v $node1
system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
sql connect
$i = 0
while $i < $num
$port = $i * 100
$port = $port + 8100
$i = $i + 1
sql create dnode $node1 port $port
endw
$i = 0
while $i < $num
$port = $i * 100
$port = $port + 8100
$i = $i + 1
sql create dnode $node2 port $port
endw
$i = 0
while $i < $num
$port = $i * 100
$port = $port + 8100
$i = $i + 1
sql create dnode $node3 port $port
endw
$i = 0
while $i < $num
$port = $i * 100
$port = $port + 8100
$i = $i + 1
sql create dnode $node4 port $port
endw
endi
endi
############### process nodes #####################
$i = 0
while $i < $num
$index = $i + 80
$port = $i * 100
$port = $port + 8100
$dnodename = dnode . $index
$i = $i + 1
if $option == 1 then
system sh/exec.sh -n $dnodename -s start
endi
if $option == 2 then
system sh/exec.sh -n $dnodename -s stop -x SIGINT
endi
if $option == 0 then
system sh/deploy.sh -n $dnodename -i 1
system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp
system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp
system sh/cfg.sh -n $dnodename -c fqdn -v $self
system sh/cfg.sh -n $dnodename -c serverPort -v $port
system sh/exec.sh -n $dnodename -s start
endi
endw
tests/test/c/create_table.c
浏览文件 @ f9d7cc44
...@@ -28,9 +28,14 @@ int32_t numOfThreads = 1; ...@@ -28,9 +28,14 @@ int32_t numOfThreads = 1;
int64_t numOfTables = 200000; int64_t numOfTables = 200000;
int32_t createTable = 1; int32_t createTable = 1;
int32_t insertData = 0; int32_t insertData = 0;
int32_t batchNum = 100; int32_t batchNumOfTbl = 100;
int32_t batchNumOfRow = 1;
int32_t numOfVgroups = 2; int32_t numOfVgroups = 2;
int32_t showTablesFlag = 0; int32_t showTablesFlag = 0;
int32_t queryFlag = 0;
int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000
typedef struct { typedef struct {
int64_t tableBeginIndex; int64_t tableBeginIndex;
...@@ -46,9 +51,9 @@ typedef struct { ...@@ -46,9 +51,9 @@ typedef struct {
pthread_t thread; pthread_t thread;
} SThreadInfo; } SThreadInfo;
//void parseArgument(int32_t argc, char *argv[]); // void parseArgument(int32_t argc, char *argv[]);
//void *threadFunc(void *param); // void *threadFunc(void *param);
//void createDbAndStb(); // void createDbAndStb();
void createDbAndStb() { void createDbAndStb() {
pPrint("start to create db and stable"); pPrint("start to create db and stable");
...@@ -64,7 +69,8 @@ void createDbAndStb() { ...@@ -64,7 +69,8 @@ void createDbAndStb() {
TAOS_RES *pRes = taos_query(con, qstr); TAOS_RES *pRes = taos_query(con, qstr);
int32_t code = taos_errno(pRes); int32_t code = taos_errno(pRes);
if (code != 0) { if (code != 0) {
pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), taos_errstr(pRes)); pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes),
taos_errstr(pRes));
exit(0); exit(0);
} }
taos_free_result(pRes); taos_free_result(pRes);
...@@ -129,10 +135,9 @@ static int64_t getResult(TAOS_RES *tres) { ...@@ -129,10 +135,9 @@ static int64_t getResult(TAOS_RES *tres) {
return numOfRows; return numOfRows;
} }
void showTables() {
void showTables() {
pPrint("start to show tables"); pPrint("start to show tables");
char qstr[32]; char qstr[128];
TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
if (con == NULL) { if (con == NULL) {
...@@ -140,9 +145,9 @@ void showTables() { ...@@ -140,9 +145,9 @@ void showTables() {
exit(1); exit(1);
} }
sprintf(qstr, "use %s", dbName); snprintf(qstr, 128, "use %s", dbName);
TAOS_RES *pRes = taos_query(con, qstr); TAOS_RES *pRes = taos_query(con, qstr);
int code = taos_errno(pRes); int code = taos_errno(pRes);
if (code != 0) { if (code != 0) {
pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
exit(1); exit(1);
...@@ -160,15 +165,14 @@ void showTables() { ...@@ -160,15 +165,14 @@ void showTables() {
int64_t totalTableNum = getResult(pRes); int64_t totalTableNum = getResult(pRes);
taos_free_result(pRes); taos_free_result(pRes);
pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC);
taos_close(con); taos_close(con);
} }
void *threadFunc(void *param) { void *threadFunc(void *param) {
SThreadInfo *pInfo = (SThreadInfo *)param; SThreadInfo *pInfo = (SThreadInfo *)param;
char *qstr = malloc(2000 * 1000); char *qstr = malloc(batchNumOfTbl * batchNumOfRow * 128);
int32_t code = 0; int32_t code = 0;
TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
...@@ -177,48 +181,48 @@ void *threadFunc(void *param) { ...@@ -177,48 +181,48 @@ void *threadFunc(void *param) {
exit(1); exit(1);
} }
//printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); // printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex,
// pInfo->tableEndIndex);
sprintf(qstr, "use %s", pInfo->dbName); sprintf(qstr, "use %s", pInfo->dbName);
TAOS_RES *pRes = taos_query(con, qstr); TAOS_RES *pRes = taos_query(con, qstr);
taos_free_result(pRes); taos_free_result(pRes);
if (createTable) { if (createTable) {
int64_t curMs = 0; int64_t curMs = 0;
int64_t beginMs = taosGetTimestampMs(); int64_t beginMs = taosGetTimestampMs();
pInfo->startMs = beginMs; pInfo->startMs = beginMs;
int64_t t = pInfo->tableBeginIndex; int64_t t = pInfo->tableBeginIndex;
for (; t <= pInfo->tableEndIndex;) { for (; t <= pInfo->tableEndIndex;) {
//int64_t batch = (pInfo->tableEndIndex - t); // int64_t batch = (pInfo->tableEndIndex - t);
//batch = MIN(batch, batchNum); // batch = MIN(batch, batchNum);
int32_t len = sprintf(qstr, "create table"); int32_t len = sprintf(qstr, "create table");
for (int32_t i = 0; i < batchNum;) { for (int32_t i = 0; i < batchNumOfTbl;) {
len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t); len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t);
t++; t++;
i++; i++;
if (t > pInfo->tableEndIndex) { if (t > pInfo->tableEndIndex) {
break; break;
} }
} }
int64_t startTs = taosGetTimestampUs(); int64_t startTs = taosGetTimestampUs();
TAOS_RES *pRes = taos_query(con, qstr); TAOS_RES *pRes = taos_query(con, qstr);
code = taos_errno(pRes); code = taos_errno(pRes);
if (code != 0) { if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) {
pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code));
} }
taos_free_result(pRes); taos_free_result(pRes);
int64_t endTs = taosGetTimestampUs(); int64_t endTs = taosGetTimestampUs();
int64_t delay = endTs - startTs; int64_t delay = endTs - startTs;
//printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay);
if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; if (delay > pInfo->maxDelay) pInfo->maxDelay = delay;
if (delay < pInfo->minDelay) pInfo->minDelay = delay; if (delay < pInfo->minDelay) pInfo->minDelay = delay;
curMs = taosGetTimestampMs(); curMs = taosGetTimestampMs();
if (curMs - beginMs > 10000) { if (curMs - beginMs > 10000) {
beginMs = curMs; beginMs = curMs;
//printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t);
printCreateProgress(pInfo, t); printCreateProgress(pInfo, t);
} }
} }
...@@ -227,32 +231,50 @@ void *threadFunc(void *param) { ...@@ -227,32 +231,50 @@ void *threadFunc(void *param) {
if (insertData) { if (insertData) {
int64_t curMs = 0; int64_t curMs = 0;
int64_t beginMs = taosGetTimestampMs();; int64_t beginMs = taosGetTimestampMs();
pInfo->startMs = beginMs;
pInfo->startMs = taosGetTimestampMs(); int64_t t = pInfo->tableBeginIndex;
for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { for (; t <= pInfo->tableEndIndex;) {
int64_t batch = (pInfo->tableEndIndex - t); // int64_t batch = (pInfo->tableEndIndex - t);
batch = MIN(batch, batchNum); // batch = MIN(batch, batchNum);
int32_t len = sprintf(qstr, "insert into"); int32_t len = sprintf(qstr, "insert into ");
for (int32_t i = 0; i < batch; ++i) {
len += sprintf(qstr + len, " t%" PRId64 " values(now, %" PRId64 ")", t + i, t + i); for (int32_t i = 0; i < batchNumOfTbl;) {
} int64_t ts = startTimestamp;
len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t);
for (int32_t j = 0; j < batchNumOfRow; j++) {
len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++);
}
t++;
i++;
if (t > pInfo->tableEndIndex) {
break;
}
}
int64_t startTs = taosGetTimestampUs();
TAOS_RES *pRes = taos_query(con, qstr); TAOS_RES *pRes = taos_query(con, qstr);
code = taos_errno(pRes); code = taos_errno(pRes);
if (code != 0) { if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) {
pError("failed to insert table t%" PRId64 ", reason:%s", t, tstrerror(code)); pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code));
} }
taos_free_result(pRes); taos_free_result(pRes);
int64_t endTs = taosGetTimestampUs();
int64_t delay = endTs - startTs;
// printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay);
if (delay > pInfo->maxDelay) pInfo->maxDelay = delay;
if (delay < pInfo->minDelay) pInfo->minDelay = delay;
curMs = taosGetTimestampMs(); curMs = taosGetTimestampMs();
if (curMs - beginMs > 10000) { if (curMs - beginMs > 10000) {
beginMs = curMs;
// printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t);
printInsertProgress(pInfo, t); printInsertProgress(pInfo, t);
} }
t += (batch - 1);
} }
printInsertProgress(pInfo, pInfo->tableEndIndex); printInsertProgress(pInfo, t);
} }
taos_close(con); taos_close(con);
...@@ -281,9 +303,13 @@ void printHelp() { ...@@ -281,9 +303,13 @@ void printHelp() {
printf("%s%s\n", indent, "-i"); printf("%s%s\n", indent, "-i");
printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData); printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData);
printf("%s%s\n", indent, "-b"); printf("%s%s\n", indent, "-b");
printf("%s%s%s%d\n", indent, indent, "batchNum, default is ", batchNum); printf("%s%s%s%d\n", indent, indent, "batchNumOfTbl, default is ", batchNumOfTbl);
printf("%s%s\n", indent, "-w"); printf("%s%s\n", indent, "-w");
printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag); printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag);
printf("%s%s\n", indent, "-q");
printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag);
printf("%s%s\n", indent, "-l");
printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow);
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
} }
...@@ -310,10 +336,15 @@ void parseArgument(int32_t argc, char *argv[]) { ...@@ -310,10 +336,15 @@ void parseArgument(int32_t argc, char *argv[]) {
} else if (strcmp(argv[i], "-i") == 0) { } else if (strcmp(argv[i], "-i") == 0) {
insertData = atoi(argv[++i]); insertData = atoi(argv[++i]);
} else if (strcmp(argv[i], "-b") == 0) { } else if (strcmp(argv[i], "-b") == 0) {
batchNum = atoi(argv[++i]); batchNumOfTbl = atoi(argv[++i]);
} else if (strcmp(argv[i], "-l") == 0) {
batchNumOfRow = atoi(argv[++i]);
} else if (strcmp(argv[i], "-w") == 0) { } else if (strcmp(argv[i], "-w") == 0) {
showTablesFlag = atoi(argv[++i]); showTablesFlag = atoi(argv[++i]);
} else if (strcmp(argv[i], "-q") == 0) {
queryFlag = atoi(argv[++i]);
} else { } else {
pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC);
} }
} }
...@@ -325,8 +356,10 @@ void parseArgument(int32_t argc, char *argv[]) { ...@@ -325,8 +356,10 @@ void parseArgument(int32_t argc, char *argv[]) {
pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC); pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC);
pPrint("%s createTable:%d %s", GREEN, createTable, NC); pPrint("%s createTable:%d %s", GREEN, createTable, NC);
pPrint("%s insertData:%d %s", GREEN, insertData, NC); pPrint("%s insertData:%d %s", GREEN, insertData, NC);
pPrint("%s batchNum:%d %s", GREEN, batchNum, NC); pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC);
pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC);
pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC);
pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC);
pPrint("%s start create table performace test %s", GREEN, NC); pPrint("%s start create table performace test %s", GREEN, NC);
} }
...@@ -335,33 +368,39 @@ int32_t main(int32_t argc, char *argv[]) { ...@@ -335,33 +368,39 @@ int32_t main(int32_t argc, char *argv[]) {
parseArgument(argc, argv); parseArgument(argc, argv);
if (showTablesFlag) { if (showTablesFlag) {
showTables(); showTables();
return 0; return 0;
}
if (queryFlag) {
//selectRowsFromTable();
return 0;
} }
createDbAndStb();
pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads); if (createTable) {
createDbAndStb();
}
pPrint("%d threads are spawned to create %" PRId64 " tables", numOfThreads, numOfTables);
pthread_attr_t thattr; pthread_attr_t thattr;
pthread_attr_init(&thattr); pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo));
//int64_t numOfTablesPerThread = numOfTables / numOfThreads; // int64_t numOfTablesPerThread = numOfTables / numOfThreads;
//numOfTables = numOfTablesPerThread * numOfThreads; // numOfTables = numOfTablesPerThread * numOfThreads;
if (numOfThreads < 1) { if (numOfThreads < 1) {
numOfThreads = 1; numOfThreads = 1;
} }
int64_t a = numOfTables / numOfThreads; int64_t a = numOfTables / numOfThreads;
if (a < 1) { if (a < 1) {
numOfThreads = numOfTables; numOfThreads = numOfTables;
a = 1; a = 1;
} }
int64_t b = 0; int64_t b = 0;
b = numOfTables % numOfThreads; b = numOfTables % numOfThreads;
...@@ -371,7 +410,7 @@ int32_t main(int32_t argc, char *argv[]) { ...@@ -371,7 +410,7 @@ int32_t main(int32_t argc, char *argv[]) {
pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1;
tableFrom = pInfo[i].tableEndIndex + 1; tableFrom = pInfo[i].tableEndIndex + 1;
pInfo[i].threadIndex = i; pInfo[i].threadIndex = i;
pInfo[i].minDelay = INT64_MAX; pInfo[i].minDelay = INT64_MAX;
strcpy(pInfo[i].dbName, dbName); strcpy(pInfo[i].dbName, dbName);
strcpy(pInfo[i].stbName, stbName); strcpy(pInfo[i].stbName, stbName);
pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i));
...@@ -390,7 +429,7 @@ int32_t main(int32_t argc, char *argv[]) { ...@@ -390,7 +429,7 @@ int32_t main(int32_t argc, char *argv[]) {
createTableSpeed += pInfo[i].createTableSpeed; createTableSpeed += pInfo[i].createTableSpeed;
if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay;
if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay;
} }
float insertDataSpeed = 0; float insertDataSpeed = 0;
...@@ -398,21 +437,17 @@ int32_t main(int32_t argc, char *argv[]) { ...@@ -398,21 +437,17 @@ int32_t main(int32_t argc, char *argv[]) {
insertDataSpeed += pInfo[i].insertDataSpeed; insertDataSpeed += pInfo[i].insertDataSpeed;
} }
pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", if (createTable) {
GREEN, pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64
numOfTables, "us %s",
createTableSpeed, GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC);
numOfThreads, }
maxDelay,
minDelay,
NC);
if (insertData) { if (insertData) {
pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed,
numOfThreads, NC); numOfThreads, NC);
} }
pthread_attr_destroy(&thattr); pthread_attr_destroy(&thattr);
free(pInfo); free(pInfo);
} }
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或