Skip to content

TDengine
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
TDengine
TDengine
“c4fd53070eddb6929d57c397e8e086341c28a5d9”上不存在“ppocr/git@gitcode.net:s920243400/PaddleOCR.git”
提交 6e358fd4 编写于 作者: H Hongze Cheng
浏览文件
操作

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/vnode

上级 72ed99e8 802f0622
显示空白变更内容
内联 并排
Showing with 3189 addition and 3215 deletion
2.0/src/query/tests/astTest.cpp
浏览文件 @ 6e358fd4
......@@ -633,3 +633,5 @@ TEST(testCase, astTest) {
// exprSerializeTest2();
}
#endif
#pragma GCC diagnostic pop
\ No newline at end of file
2.0/src/query/tests/cSortTest.cpp
浏览文件 @ 6e358fd4
......@@ -5,6 +5,7 @@
#include "tsdb.h"
#include "qExtbuffer.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
......@@ -122,3 +123,5 @@ TEST(testCase, columnsort_test) {
destroyColumnModel(pModel);
}
#pragma GCC diagnostic pop
\ No newline at end of file
2.0/src/query/tests/histogramTest.cpp
浏览文件 @ 6e358fd4
......@@ -6,6 +6,7 @@
#include "taos.h"
#include "qHistogram.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
......@@ -140,3 +141,5 @@ TEST(testCase, heapsort) {
//
// free(pEntry);
}
#pragma GCC diagnostic pop
\ No newline at end of file
2.0/src/query/tests/patternMatchTest.cpp
浏览文件 @ 6e358fd4
......@@ -6,6 +6,7 @@
#include "qAggMain.h"
#include "tcompare.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
......@@ -84,3 +85,5 @@ TEST(testCase, patternMatchTest) {
ret = patternMatch("%9", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
}
#pragma GCC diagnostic pop
\ No newline at end of file
2.0/src/query/tests/percentileTest.cpp
浏览文件 @ 6e358fd4
......@@ -7,6 +7,7 @@
#include "qPercentile.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
......@@ -255,3 +256,5 @@ TEST(testCase, percentileTest) {
unsignedDataTest();
largeDataTest();
}
#pragma GCC diagnostic pop
\ No newline at end of file
2.0/src/query/tests/rangeMergeTest.cpp
浏览文件 @ 6e358fd4
......@@ -7,6 +7,7 @@
#include "qFilter.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
......@@ -365,3 +366,5 @@ TEST(testCase, rangeMergeTest) {
intDataTest();
}
#pragma GCC diagnostic pop
\ No newline at end of file
2.0/src/query/tests/resultBufferTest.cpp
浏览文件 @ 6e358fd4
......@@ -6,6 +6,7 @@
#include "taos.h"
#include "tsdb.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
......@@ -161,3 +162,5 @@ TEST(testCase, resultBufferTest) {
writeDownTest();
recyclePageTest();
}
#pragma GCC diagnostic pop
\ No newline at end of file
2.0/src/query/tests/tsBufTest.cpp
浏览文件 @ 6e358fd4
......@@ -9,6 +9,7 @@
#include "ttoken.h"
#include "tutil.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
......@@ -513,3 +514,5 @@ TEST(testCase, tsBufTest) {
mergeDiffVnodeBufferTest();
mergeIdenticalVnodeBufferTest();
}
#pragma GCC diagnostic pop
\ No newline at end of file
2.0/src/query/tests/unitTest.cpp
浏览文件 @ 6e358fd4
......@@ -4,6 +4,7 @@
#include "taos.h"
#include "tsdb.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
......@@ -910,3 +911,4 @@ TEST(testCase, getTempFilePath_test) {
printf("%s\n", path);
}
#pragma GCC diagnostic pop
\ No newline at end of file
CMakeLists.txt
浏览文件 @ 6e358fd4
......@@ -10,8 +10,8 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
include(${CMAKE_SUPPORT_DIR}/cmake.options)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
# contrib
add_subdirectory(contrib)
......
contrib/test/traft/CMakeLists.txt
浏览文件 @ 6e358fd4
add_subdirectory(rebalance_leader)
add_subdirectory(make_cluster)
add_subdirectory(single_node)
contrib/test/traft/cluster/Makefile.2 0 → 100644
浏览文件 @ 6e358fd4
all:
gcc node10000.c -I ../../include/ ../../.libs/libraft.a -o node10000 -luv -llz4 -lpthread -g
gcc node10001.c -I ../../include/ ../../.libs/libraft.a -o node10001 -luv -llz4 -lpthread -g
gcc node10002.c -I ../../include/ ../../.libs/libraft.a -o node10002 -luv -llz4 -lpthread -g
gcc node10000_restart.c -I ../../include/ ../../.libs/libraft.a -o node10000_restart -luv -llz4 -lpthread -g
gcc node10001_restart.c -I ../../include/ ../../.libs/libraft.a -o node10001_restart -luv -llz4 -lpthread -g
gcc node10002_restart.c -I ../../include/ ../../.libs/libraft.a -o node10002_restart -luv -llz4 -lpthread -g
clean:
rm -f node10000
rm -f node10001
rm -f node10002
rm -f node10000_restart
rm -f node10001_restart
rm -f node10002_restart
sh clear.sh
contrib/test/traft/cluster/node10000.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
// printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// add vgroup, id = 100, only has 3 replica.
// array <peers, peersCount> gives the peer replica infomation.
char peers[MAX_PEERS_COUNT][ADDRESS_LEN];
memset(peers, 0, sizeof(peers));
snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10001");
snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10002");
uint32_t peersCount = 2;
r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm);
assert(r == 0);
// for test: submit a value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/cluster/node10000_restart.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
// printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// add vgroup, id = 100, only has 3 replica.
// here only add self.
// peer replica information will restore from wal.
r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm);
assert(r == 0);
// for test: submit a value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/cluster/node10001.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
// printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// add vgroup, id = 100, only has 3 replica.
// array <peers, peersCount> gives the peer replica infomation.
char peers[MAX_PEERS_COUNT][ADDRESS_LEN];
memset(peers, 0, sizeof(peers));
snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10000");
snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10002");
uint32_t peersCount = 2;
r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm);
assert(r == 0);
// for test: submit a value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/cluster/node10001_restart.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
// printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// add vgroup, id = 100, only has 3 replica.
// here only add self.
// peer replica information will restore from wal.
r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm);
assert(r == 0);
// for test: submit a value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/cluster/node10002.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
// printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// add vgroup, id = 100, only has 3 replica.
// array <peers, peersCount> gives the peer replica infomation.
char peers[MAX_PEERS_COUNT][ADDRESS_LEN];
memset(peers, 0, sizeof(peers));
snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10000");
snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10001");
uint32_t peersCount = 2;
r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm);
assert(r == 0);
// for test: submit a value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/cluster/node10002_restart.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
// printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// add vgroup, id = 100, only has 3 replica.
// here only add self.
// peer replica information will restore from wal.
r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm);
assert(r == 0);
// for test: submit a value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/join_into_vgroup/Makefile.2 0 → 100644
浏览文件 @ 6e358fd4
all:
gcc node_follower10000.c -I ../../include/ ../../.libs/libraft.a -o node_follower10000 -luv -llz4 -lpthread -g
gcc node_follower10001.c -I ../../include/ ../../.libs/libraft.a -o node_follower10001 -luv -llz4 -lpthread -g
gcc node_leader10002.c -I ../../include/ ../../.libs/libraft.a -o node_leader10002 -luv -llz4 -lpthread -g
clean:
rm -f node_follower10000
rm -f node_follower10001
rm -f node_leader10002
sh clear.sh
contrib/test/traft/join_into_vgroup/node_follower10000.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// add one replica
r = addRaftSpare(&raftEnv, 100, pFsm);
assert(r == 0);
// for test: submit value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/join_into_vgroup/node_follower10001.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// add one replica
r = addRaftSpare(&raftEnv, 100, pFsm);
assert(r == 0);
// for test: submit value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/join_into_vgroup/node_leader10002.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
void joinRaftPeerCb(struct raft_change *req, int status) {
struct raft *r = req->data;
if (status != 0) {
fprintf(stderr, "joinRaftPeerCb error: %s \n", raft_errmsg(r));
} else {
fprintf(stderr, "joinRaftPeerCb ok \n");
}
raft_free(req);
}
int main(int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// add one replica
r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm);
assert(r == 0);
printRaftState(getRaft(&raftEnv, 100));
// wait for being leader
while (1) {
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
if (r->state == RAFT_LEADER) {
break;
}
}
// join peers
r = joinRaftPeer(&raftEnv, 100, "127.0.0.1", 10000, joinRaftPeerCb);
assert(r == 0);
// wait for join over
sleep(2);
r = joinRaftPeer(&raftEnv, 100, "127.0.0.1", 10001, joinRaftPeerCb);
assert(r == 0);
// for test: submit value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
contrib/test/traft/make_cluster/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ 72ed99e8
add_executable(makeCluster "")
target_sources(makeCluster
PRIVATE
"raftMain.c"
"raftServer.c"
"config.c"
"console.c"
"simpleHash.c"
"util.c"
)
target_link_libraries(makeCluster PUBLIC traft lz4 uv_a)
contrib/test/traft/make_cluster/common.h 已删除 100644 → 0
浏览文件 @ 72ed99e8
#ifndef TRAFT_COMMON_H
#define TRAFT_COMMON_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#define COMMAND_LEN 512
#define MAX_CMD_COUNT 10
#define TOKEN_LEN 128
#define MAX_PEERS_COUNT 19
#define HOST_LEN 64
#define ADDRESS_LEN (HOST_LEN * 2)
#define BASE_DIR_LEN 128
#ifdef __cplusplus
}
#endif
#endif
contrib/test/traft/make_cluster/config.c 已删除 100644 → 0
浏览文件 @ 72ed99e8
#include "config.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
void addrToString(const char *host, uint16_t port, char *addr, int len) { snprintf(addr, len, "%s:%hu", host, port); }
void parseAddr(const char *addr, char *host, int len, uint16_t *port) {
char *tmp = (char *)malloc(strlen(addr) + 1);
strcpy(tmp, addr);
char *context;
char *separator = ":";
char *token = strtok_r(tmp, separator, &context);
if (token) {
snprintf(host, len, "%s", token);
}
token = strtok_r(NULL, separator, &context);
if (token) {
sscanf(token, "%hu", port);
}
free(tmp);
}
int parseConf(int argc, char **argv, RaftServerConfig *pConf) {
memset(pConf, 0, sizeof(*pConf));
int option_index, option_value;
option_index = 0;
static struct option long_options[] = {{"help", no_argument, NULL, 'h'},
{"addr", required_argument, NULL, 'a'},
{"dir", required_argument, NULL, 'd'},
{NULL, 0, NULL, 0}};
while ((option_value = getopt_long(argc, argv, "ha:d:", long_options, &option_index)) != -1) {
switch (option_value) {
case 'a': {
parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port);
break;
}
case 'd': {
snprintf(pConf->baseDir, sizeof(pConf->baseDir), "%s", optarg);
break;
}
case 'h': {
return -2;
}
default: { return -2; }
}
}
return 0;
}
void printConf(RaftServerConfig *pConf) {
printf("\n---printConf: \n");
printf("me: [%s:%hu] \n", pConf->me.host, pConf->me.port);
printf("dataDir: [%s] \n\n", pConf->baseDir);
}
contrib/test/traft/make_cluster/config.h 已删除 100644 → 0
浏览文件 @ 72ed99e8
#ifndef TRAFT_CONFIG_H
#define TRAFT_CONFIG_H
#ifdef __cplusplus
extern "C" {
#endif
#include <getopt.h>
#include <stdint.h>
#include "common.h"
typedef struct {
char host[HOST_LEN];
uint16_t port;
} Addr;
typedef struct {
Addr me;
char baseDir[BASE_DIR_LEN];
} RaftServerConfig;
void addrToString(const char *host, uint16_t port, char *addr, int len);
void parseAddr(const char *addr, char *host, int len, uint16_t *port);
int parseConf(int argc, char **argv, RaftServerConfig *pConf);
void printConf(RaftServerConfig *pConf);
#ifdef __cplusplus
}
#endif
#endif
contrib/test/traft/make_cluster/console.c 已删除 100644 → 0
浏览文件 @ 72ed99e8
#include "console.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "raftServer.h"
#include "util.h"
void printHelp() {
printf("---------------------\n");
printf("help: \n\n");
printf("create a vgroup with 3 replicas: \n");
printf("create vnode voter vid 100 peers 127.0.0.1:10001 127.0.0.1:10002 \n");
printf("create vnode voter vid 100 peers 127.0.0.1:10000 127.0.0.1:10002 \n");
printf("create vnode voter vid 100 peers 127.0.0.1:10000 127.0.0.1:10001 \n");
printf("\n");
printf("create a vgroup with only one replica: \n");
printf("create vnode voter vid 200 \n");
printf("\n");
printf("add vnode into vgroup: \n");
printf("create vnode spare vid 100 ---- run at 127.0.0.1:10003\n");
printf("join vnode vid 100 addr 127.0.0.1:10003 ---- run at leader of vgroup 100\n");
printf("\n");
printf("run \n");
printf("put 0 key value \n");
printf("get 0 key \n");
printf("---------------------\n");
}
void console(RaftServer *pRaftServer) {
while (1) {
int ret;
char cmdBuf[COMMAND_LEN];
memset(cmdBuf, 0, sizeof(cmdBuf));
printf("(console)> ");
char *retp = fgets(cmdBuf, COMMAND_LEN, stdin);
if (!retp) {
exit(-1);
}
int pos = strlen(cmdBuf);
if (cmdBuf[pos - 1] == '\n') {
cmdBuf[pos - 1] = '\0';
}
if (strncmp(cmdBuf, "", COMMAND_LEN) == 0) {
continue;
}
char cmds[MAX_CMD_COUNT][TOKEN_LEN];
memset(cmds, 0, sizeof(cmds));
int cmdCount;
cmdCount = splitString(cmdBuf, " ", cmds, MAX_CMD_COUNT);
if (strcmp(cmds[0], "create") == 0 && strcmp(cmds[1], "vnode") == 0 && strcmp(cmds[3], "vid") == 0) {
uint16_t vid;
sscanf(cmds[4], "%hu", &vid);
if (strcmp(cmds[2], "voter") == 0) {
char peers[MAX_PEERS_COUNT][ADDRESS_LEN];
memset(peers, 0, sizeof(peers));
uint32_t peersCount = 0;
if (strcmp(cmds[5], "peers") == 0 && cmdCount > 6) {
// create vnode voter vid 100 peers 127.0.0.1:10001 127.0.0.1:10002
for (int i = 6; i < cmdCount; ++i) {
snprintf(peers[i - 6], ADDRESS_LEN, "%s", cmds[i]);
peersCount++;
}
} else {
// create vnode voter vid 200
}
ret = addRaftVoter(pRaftServer, peers, peersCount, vid);
if (ret == 0) {
printf("create vnode voter ok \n");
} else {
printf("create vnode voter error \n");
}
} else if (strcmp(cmds[2], "spare") == 0) {
ret = addRaftSpare(pRaftServer, vid);
if (ret == 0) {
printf("create vnode spare ok \n");
} else {
printf("create vnode spare error \n");
}
} else {
printHelp();
}
} else if (strcmp(cmds[0], "join") == 0 && strcmp(cmds[1], "vnode") == 0 && strcmp(cmds[2], "vid") == 0 &&
strcmp(cmds[4], "addr") == 0 && cmdCount == 6) {
// join vnode vid 100 addr 127.0.0.1:10004
char * address = cmds[5];
char host[64];
uint16_t port;
parseAddr(address, host, sizeof(host), &port);
uint16_t vid;
sscanf(cmds[3], "%hu", &vid);
HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid);
if (*pp == NULL) {
printf("vid:%hu not found \n", vid);
break;
}
RaftInstance *pRaftInstance = (*pp)->data;
uint64_t destRaftId = encodeRaftId(host, port, vid);
struct raft_change *req = raft_malloc(sizeof(*req));
RaftJoin * pRaftJoin = raft_malloc(sizeof(*pRaftJoin));
pRaftJoin->r = &pRaftInstance->raft;
pRaftJoin->joinId = destRaftId;
req->data = pRaftJoin;
ret = raft_add(&pRaftInstance->raft, req, destRaftId, address, raftChangeAddCb);
if (ret != 0) {
printf("raft_add error: %s \n", raft_errmsg(&pRaftInstance->raft));
}
} else if (strcmp(cmds[0], "dropnode") == 0) {
} else if (strcmp(cmds[0], "state") == 0) {
pRaftServer->raftInstances.print(&pRaftServer->raftInstances);
for (size_t i = 0; i < pRaftServer->raftInstances.length; ++i) {
HashNode *ptr = pRaftServer->raftInstances.table[i];
if (ptr != NULL) {
while (ptr != NULL) {
RaftInstance *pRaftInstance = ptr->data;
printf("instance vid:%hu raftId:%llu \n", ptr->vgroupId, pRaftInstance->raftId);
printRaftState(&pRaftInstance->raft);
printf("\n");
ptr = ptr->next;
}
printf("\n");
}
}
} else if (strcmp(cmds[0], "put") == 0 && cmdCount == 4) {
uint16_t vid;
sscanf(cmds[1], "%hu", &vid);
char * key = cmds[2];
char * value = cmds[3];
HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid);
if (*pp == NULL) {
printf("vid:%hu not found \n", vid);
break;
}
RaftInstance *pRaftInstance = (*pp)->data;
char *raftValue = malloc(TOKEN_LEN * 2 + 3);
snprintf(raftValue, TOKEN_LEN * 2 + 3, "%s--%s", key, value);
putValue(&pRaftInstance->raft, raftValue);
free(raftValue);
} else if (strcmp(cmds[0], "run") == 0) {
pthread_t tidRaftServer;
pthread_create(&tidRaftServer, NULL, startServerFunc, pRaftServer);
} else if (strcmp(cmds[0], "get") == 0 && cmdCount == 3) {
uint16_t vid;
sscanf(cmds[1], "%hu", &vid);
char * key = cmds[2];
HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid);
if (*pp == NULL) {
printf("vid:%hu not found \n", vid);
break;
}
RaftInstance * pRaftInstance = (*pp)->data;
SimpleHash * pKV = pRaftInstance->fsm.data;
SimpleHashNode **ppNode = pKV->find_cstr(pKV, key);
if (*ppNode == NULL) {
printf("key:%s not found \n", key);
} else {
printf("find key:%s value:%s \n", key, (char *)((*ppNode)->data));
}
} else if (strcmp(cmds[0], "transfer") == 0) {
} else if (strcmp(cmds[0], "state") == 0) {
} else if (strcmp(cmds[0], "snapshot") == 0) {
} else if (strcmp(cmds[0], "exit") == 0) {
exit(0);
} else if (strcmp(cmds[0], "quit") == 0) {
exit(0);
} else if (strcmp(cmds[0], "help") == 0) {
printHelp();
} else {
printf("unknown command: %s \n", cmdBuf);
printHelp();
}
/*
printf("cmdBuf: [%s] \n", cmdBuf);
printf("cmdCount : %d \n", cmdCount);
for (int i = 0; i < MAX_CMD_COUNT; ++i) {
printf("cmd%d : %s \n", i, cmds[i]);
}
*/
}
}
contrib/test/traft/make_cluster/console.h 已删除 100644 → 0
浏览文件 @ 72ed99e8
#ifndef TRAFT_CONSOLE_H
#define TRAFT_CONSOLE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <getopt.h>
#include <stdint.h>
#include "common.h"
#include "raftServer.h"
void console(RaftServer *pRaftServer);
#ifdef __cplusplus
}
#endif
#endif
contrib/test/traft/make_cluster/raftMain.c 已删除 100644 → 0
浏览文件 @ 72ed99e8
#include <assert.h>
#include <getopt.h>
#include <pthread.h>
#include <raft.h>
#include <raft/uv.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include "common.h"
#include "config.h"
#include "console.h"
#include "raftServer.h"
#include "simpleHash.h"
#include "util.h"
const char *exe_name;
void *startConsoleFunc(void *param) {
RaftServer *pRaftServer = (RaftServer *)param;
console(pRaftServer);
return NULL;
}
void usage() {
printf("\nusage: \n");
printf("%s --addr=127.0.0.1:10000 --dir=./data \n", exe_name);
printf("\n");
}
RaftServerConfig gConfig;
RaftServer gRaftServer;
int main(int argc, char **argv) {
srand(time(NULL));
int32_t ret;
exe_name = argv[0];
if (argc < 3) {
usage();
exit(-1);
}
ret = parseConf(argc, argv, &gConfig);
if (ret != 0) {
usage();
exit(-1);
}
printConf(&gConfig);
if (!dirOK(gConfig.baseDir)) {
ret = mkdir(gConfig.baseDir, 0775);
if (ret != 0) {
fprintf(stderr, "mkdir error, %s \n", gConfig.baseDir);
exit(-1);
}
}
ret = raftServerInit(&gRaftServer, &gConfig);
if (ret != 0) {
fprintf(stderr, "raftServerInit error \n");
exit(-1);
}
/*
pthread_t tidRaftServer;
pthread_create(&tidRaftServer, NULL, startServerFunc, &gRaftServer);
*/
pthread_t tidConsole;
pthread_create(&tidConsole, NULL, startConsoleFunc, &gRaftServer);
while (1) {
sleep(10);
}
return 0;
}
contrib/test/traft/make_cluster/raftServer.c 已删除 100644 → 0
浏览文件 @ 72ed99e8
#include "raftServer.h"
#include <stdlib.h>
#include <unistd.h>
#include "common.h"
#include "simpleHash.h"
#include "util.h"
void *startServerFunc(void *param) {
RaftServer *pRaftServer = (RaftServer *)param;
int32_t r = raftServerStart(pRaftServer);
assert(r == 0);
return NULL;
}
void raftChangeAssignCb(struct raft_change *req, int status) {
struct raft *r = req->data;
if (status != 0) {
printf("raftChangeAssignCb error: %s \n", raft_errmsg(r));
} else {
printf("raftChangeAssignCb ok \n");
}
raft_free(req);
}
void raftChangeAddCb(struct raft_change *req, int status) {
RaftJoin *pRaftJoin = req->data;
if (status != 0) {
printf("raftChangeAddCb error: %s \n", raft_errmsg(pRaftJoin->r));
} else {
struct raft_change *req2 = raft_malloc(sizeof(*req2));
req2->data = pRaftJoin->r;
int ret = raft_assign(pRaftJoin->r, req2, pRaftJoin->joinId, RAFT_VOTER, raftChangeAssignCb);
if (ret != 0) {
printf("raftChangeAddCb error: %s \n", raft_errmsg(pRaftJoin->r));
}
}
raft_free(req->data);
raft_free(req);
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) {
// get fsm data
SimpleHash *sh = pFsm->data;
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: [%s] \n", msg);
char arr[2][TOKEN_LEN];
int r = splitString(msg, "--", arr, 2);
assert(r == 2);
// do the value on fsm
sh->insert_cstr(sh, arr[0], arr[1]);
raft_free(buf->base);
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
struct raft *r = req->data;
if (status != 0) {
printf("putValueCb error: %s \n", raft_errmsg(r));
} else {
printf("putValueCb: %s \n", "ok");
}
raft_free(req);
}
void putValue(struct raft *r, const char *value) {
struct raft_buffer buf;
buf.len = strlen(value) + 1;
buf.base = raft_malloc(buf.len);
snprintf(buf.base, buf.len, "%s", value);
struct raft_apply *req = raft_malloc(sizeof(*req));
req->data = r;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
const char *state2String(unsigned short state) {
if (state == RAFT_UNAVAILABLE) {
return "RAFT_UNAVAILABLE";
} else if (state == RAFT_FOLLOWER) {
return "RAFT_FOLLOWER";
} else if (state == RAFT_CANDIDATE) {
return "RAFT_CANDIDATE";
} else if (state == RAFT_LEADER) {
return "RAFT_LEADER";
}
return "UNKNOWN_RAFT_STATE";
}
void printRaftConfiguration(struct raft_configuration *c) {
printf("configuration: \n");
for (int i = 0; i < c->n; ++i) {
printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address);
}
}
void printRaftState(struct raft *r) {
printf("----Raft State: -----------\n");
printf("mem_addr: %p \n", r);
printf("my_id: %llu \n", r->id);
printf("address: %s \n", r->address);
printf("current_term: %llu \n", r->current_term);
printf("voted_for: %llu \n", r->voted_for);
printf("role: %s \n", state2String(r->state));
printf("commit_index: %llu \n", r->commit_index);
printf("last_applied: %llu \n", r->last_applied);
printf("last_stored: %llu \n", r->last_stored);
printf("configuration_index: %llu \n", r->configuration_index);
printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index);
printRaftConfiguration(&r->configuration);
printf("----------------------------\n");
}
int32_t addRaftVoter(RaftServer *pRaftServer, char peers[][ADDRESS_LEN], uint32_t peersCount, uint16_t vid) {
int ret;
RaftInstance *pRaftInstance = malloc(sizeof(*pRaftInstance));
assert(pRaftInstance != NULL);
// init raftId
pRaftInstance->raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid);
// init dir
snprintf(pRaftInstance->dir, sizeof(pRaftInstance->dir), "%s/%s_%hu_%hu_%llu", pRaftServer->baseDir,
pRaftServer->host, pRaftServer->port, vid, pRaftInstance->raftId);
if (!dirOK(pRaftInstance->dir)) {
ret = mkdir(pRaftInstance->dir, 0775);
if (ret != 0) {
fprintf(stderr, "mkdir error, %s \n", pRaftInstance->dir);
assert(0);
}
}
// init fsm
pRaftInstance->fsm.data = newSimpleHash(2);
pRaftInstance->fsm.apply = fsmApplyCb;
// init io
ret = raft_uv_init(&pRaftInstance->io, &pRaftServer->loop, pRaftInstance->dir, &pRaftServer->transport);
if (ret != 0) {
fprintf(stderr, "raft_uv_init error, %s \n", raft_errmsg(&pRaftInstance->raft));
assert(0);
}
// init raft
ret = raft_init(&pRaftInstance->raft, &pRaftInstance->io, &pRaftInstance->fsm, pRaftInstance->raftId,
pRaftServer->address);
if (ret != 0) {
fprintf(stderr, "raft_init error, %s \n", raft_errmsg(&pRaftInstance->raft));
assert(0);
}
// init raft_configuration
struct raft_configuration conf;
raft_configuration_init(&conf);
raft_configuration_add(&conf, pRaftInstance->raftId, pRaftServer->address, RAFT_VOTER);
for (int i = 0; i < peersCount; ++i) {
char * peerAddress = peers[i];
char host[64];
uint16_t port;
parseAddr(peerAddress, host, sizeof(host), &port);
uint64_t raftId = encodeRaftId(host, port, vid);
raft_configuration_add(&conf, raftId, peers[i], RAFT_VOTER);
}
raft_bootstrap(&pRaftInstance->raft, &conf);
// start raft
ret = raft_start(&pRaftInstance->raft);
if (ret != 0) {
fprintf(stderr, "raft_start error, %s \n", raft_errmsg(&pRaftInstance->raft));
assert(0);
}
// add raft instance into raft server
pRaftServer->raftInstances.insert(&pRaftServer->raftInstances, vid, pRaftInstance);
return 0;
}
int32_t addRaftSpare(RaftServer *pRaftServer, uint16_t vid) {
int ret;
RaftInstance *pRaftInstance = malloc(sizeof(*pRaftInstance));
assert(pRaftInstance != NULL);
// init raftId
pRaftInstance->raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid);
// init dir
snprintf(pRaftInstance->dir, sizeof(pRaftInstance->dir), "%s/%s_%hu_%hu_%llu", pRaftServer->baseDir,
pRaftServer->host, pRaftServer->port, vid, pRaftInstance->raftId);
ret = mkdir(pRaftInstance->dir, 0775);
if (ret != 0) {
fprintf(stderr, "mkdir error, %s \n", pRaftInstance->dir);
assert(0);
}
// init fsm
pRaftInstance->fsm.data = newSimpleHash(2);
pRaftInstance->fsm.apply = fsmApplyCb;
// init io
ret = raft_uv_init(&pRaftInstance->io, &pRaftServer->loop, pRaftInstance->dir, &pRaftServer->transport);
if (ret != 0) {
fprintf(stderr, "raft_uv_init error, %s \n", raft_errmsg(&pRaftInstance->raft));
assert(0);
}
// init raft
ret = raft_init(&pRaftInstance->raft, &pRaftInstance->io, &pRaftInstance->fsm, pRaftInstance->raftId,
pRaftServer->address);
if (ret != 0) {
fprintf(stderr, "raft_init error, %s \n", raft_errmsg(&pRaftInstance->raft));
assert(0);
}
// init raft_configuration
struct raft_configuration conf;
raft_configuration_init(&conf);
raft_configuration_add(&conf, pRaftInstance->raftId, pRaftServer->address, RAFT_SPARE);
raft_bootstrap(&pRaftInstance->raft, &conf);
// start raft
ret = raft_start(&pRaftInstance->raft);
if (ret != 0) {
fprintf(stderr, "raft_start error, %s \n", raft_errmsg(&pRaftInstance->raft));
assert(0);
}
// add raft instance into raft server
pRaftServer->raftInstances.insert(&pRaftServer->raftInstances, vid, pRaftInstance);
return 0;
}
int32_t raftServerInit(RaftServer *pRaftServer, const RaftServerConfig *pConf) {
int ret;
// init host, port, address, dir
snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host);
pRaftServer->port = pConf->me.port;
snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port);
snprintf(pRaftServer->baseDir, sizeof(pRaftServer->baseDir), "%s", pConf->baseDir);
// init loop
ret = uv_loop_init(&pRaftServer->loop);
if (ret != 0) {
fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret));
assert(0);
}
// init network
ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop);
if (ret != 0) {
fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret);
assert(0);
}
// init raft instance container
initIdHash(&pRaftServer->raftInstances, 2);
return 0;
}
int32_t raftServerStart(RaftServer *pRaftServer) {
// start loop
uv_run(&pRaftServer->loop, UV_RUN_DEFAULT);
return 0;
}
void raftServerStop(RaftServer *pRaftServer) {}
contrib/test/traft/make_cluster/raftServer.h 已删除 100644 → 0
浏览文件 @ 72ed99e8
#ifndef TDENGINE_RAFT_SERVER_H
#define TDENGINE_RAFT_SERVER_H
#ifdef __cplusplus
extern "C" {
#endif
#include <arpa/inet.h>
#include <assert.h>
#include <netinet/in.h>
#include <string.h>
#include "common.h"
#include "config.h"
#include "raft.h"
#include "raft/uv.h"
#include "simpleHash.h"
typedef struct RaftJoin {
struct raft *r;
raft_id joinId;
} RaftJoin;
typedef struct {
raft_id raftId;
char dir[BASE_DIR_LEN * 2];
struct raft_fsm fsm;
struct raft_io io;
struct raft raft;
} RaftInstance;
typedef struct {
char host[HOST_LEN];
uint16_t port;
char address[ADDRESS_LEN]; /* Raft instance address */
char baseDir[BASE_DIR_LEN]; /* Raft instance address */
struct uv_loop_s loop; /* UV loop */
struct raft_uv_transport transport; /* UV I/O backend transport */
IdHash raftInstances; /* multi raft instances. traft use IdHash to manager multi vgroup inside, here we can use IdHash
too. */
} RaftServer;
void * startServerFunc(void *param);
int32_t addRaftVoter(RaftServer *pRaftServer, char peers[][ADDRESS_LEN], uint32_t peersCount, uint16_t vid);
int32_t addRaftSpare(RaftServer *pRaftServer, uint16_t vid);
int32_t raftServerInit(RaftServer *pRaftServer, const RaftServerConfig *pConf);
int32_t raftServerStart(RaftServer *pRaftServer);
void raftServerStop(RaftServer *pRaftServer);
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result);
void putValueCb(struct raft_apply *req, int status, void *result);
void putValue(struct raft *r, const char *value);
void raftChangeAddCb(struct raft_change *req, int status);
const char *state2String(unsigned short state);
void printRaftConfiguration(struct raft_configuration *c);
void printRaftState(struct raft *r);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_RAFT_SERVER_H
contrib/test/traft/make_cluster/simpleHash.c 已删除 100644 → 0
浏览文件 @ 72ed99e8
#include "simpleHash.h"
uint32_t mySimpleHash(const char* data, size_t n, uint32_t seed) {
// Similar to murmur hash
const uint32_t m = 0xc6a4a793;
const uint32_t r = 24;
const char* limit = data + n;
uint32_t h = seed ^ (n * m);
// Pick up four bytes at a time
while (data + 4 <= limit) {
// uint32_t w = DecodeFixed32(data);
uint32_t w;
memcpy(&w, data, 4);
data += 4;
h += w;
h *= m;
h ^= (h >> 16);
}
// Pick up remaining bytes
switch (limit - data) {
case 3:
h += (unsigned char)(data[2]) << 16;
do {
} while (0);
case 2:
h += (unsigned char)(data[1]) << 8;
do {
} while (0);
case 1:
h += (unsigned char)(data[0]);
h *= m;
h ^= (h >> r);
break;
}
return h;
}
int insertCStrSimpleHash(struct SimpleHash* ths, char* key, char* data) {
return insertSimpleHash(ths, key, strlen(key) + 1, data, strlen(data) + 1);
}
int removeCStrSimpleHash(struct SimpleHash* ths, char* key) { return removeSimpleHash(ths, key, strlen(key) + 1); }
SimpleHashNode** findCStrSimpleHash(struct SimpleHash* ths, char* key) {
return findSimpleHash(ths, key, strlen(key) + 1);
}
int insertSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen) {
SimpleHashNode** pp = ths->find(ths, key, keyLen);
if (*pp != NULL) {
fprintf(stderr, "insertSimpleHash, already has key \n");
return -1;
}
SimpleHashNode* node = malloc(sizeof(*node));
node->hashCode = ths->hashFunc(key, keyLen);
node->key = malloc(keyLen);
node->keyLen = keyLen;
memcpy(node->key, key, keyLen);
node->data = malloc(dataLen);
node->dataLen = dataLen;
memcpy(node->data, data, dataLen);
node->next = NULL;
// printf("insertSimpleHash: <%s, %ld, %s, %ld, %u> \n", node->key, node->keyLen, node->data, node->dataLen,
// node->hashCode);
size_t index = node->hashCode & (ths->length - 1);
SimpleHashNode* ptr = ths->table[index];
if (ptr != NULL) {
node->next = ptr;
ths->table[index] = node;
} else {
ths->table[index] = node;
}
ths->elems++;
if (ths->elems > 2 * ths->length) {
ths->resize(ths);
}
return 0;
}
int removeSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen) {
SimpleHashNode** pp = ths->find(ths, key, keyLen);
if (*pp == NULL) {
fprintf(stderr, "removeSimpleHash, key not exist \n");
return -1;
}
SimpleHashNode* del = *pp;
*pp = del->next;
free(del->key);
free(del->data);
free(del);
ths->elems--;
return 0;
}
SimpleHashNode** findSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen) {
uint32_t hashCode = ths->hashFunc(key, keyLen);
// size_t index = hashCode % ths->length;
size_t index = hashCode & (ths->length - 1);
// printf("findSimpleHash: %s %ld %u \n", key, keyLen, hashCode);
SimpleHashNode** pp = &(ths->table[index]);
while (*pp != NULL && ((*pp)->hashCode != hashCode || memcmp(key, (*pp)->key, keyLen) != 0)) {
pp = &((*pp)->next);
}
return pp;
}
void printCStrSimpleHash(struct SimpleHash* ths) {
printf("\n--- printCStrSimpleHash: elems:%d length:%d \n", ths->elems, ths->length);
for (size_t i = 0; i < ths->length; ++i) {
SimpleHashNode* ptr = ths->table[i];
if (ptr != NULL) {
printf("%zu: ", i);
while (ptr != NULL) {
printf("<%u, %s, %ld, %s, %ld> ", ptr->hashCode, (char*)ptr->key, ptr->keyLen, (char*)ptr->data, ptr->dataLen);
ptr = ptr->next;
}
printf("\n");
}
}
printf("---------------\n");
}
void destroySimpleHash(struct SimpleHash* ths) {
for (size_t i = 0; i < ths->length; ++i) {
SimpleHashNode* ptr = ths->table[i];
while (ptr != NULL) {
SimpleHashNode* tmp = ptr;
ptr = ptr->next;
free(tmp->key);
free(tmp->data);
free(tmp);
}
}
ths->length = 0;
ths->elems = 0;
free(ths->table);
free(ths);
}
void resizeSimpleHash(struct SimpleHash* ths) {
uint32_t new_length = ths->length;
while (new_length < ths->elems) {
new_length *= 2;
}
printf("resizeSimpleHash: %p from %u to %u \n", ths, ths->length, new_length);
SimpleHashNode** new_table = malloc(new_length * sizeof(SimpleHashNode*));
memset(new_table, 0, new_length * sizeof(SimpleHashNode*));
uint32_t count = 0;
for (uint32_t i = 0; i < ths->length; i++) {
if (ths->table[i] == NULL) {
continue;
}
SimpleHashNode* it = ths->table[i];
while (it != NULL) {
SimpleHashNode* move_node = it;
it = it->next;
// move move_node
move_node->next = NULL;
size_t index = move_node->hashCode & (new_length - 1);
SimpleHashNode* ptr = new_table[index];
if (ptr != NULL) {
move_node->next = ptr;
new_table[index] = move_node;
} else {
new_table[index] = move_node;
}
count++;
}
}
assert(ths->elems == count);
free(ths->table);
ths->table = new_table;
ths->length = new_length;
}
uint32_t simpleHashFunc(const char* key, size_t keyLen) { return mySimpleHash(key, keyLen, 1); }
struct SimpleHash* newSimpleHash(size_t length) {
struct SimpleHash* ths = malloc(sizeof(*ths));
ths->length = length;
ths->elems = 0;
ths->table = malloc(length * sizeof(SimpleHashNode*));
memset(ths->table, 0, length * sizeof(SimpleHashNode*));
ths->insert = insertSimpleHash;
ths->remove = removeSimpleHash;
ths->find = findSimpleHash;
ths->insert_cstr = insertCStrSimpleHash;
ths->remove_cstr = removeCStrSimpleHash;
ths->find_cstr = findCStrSimpleHash;
ths->print_cstr = printCStrSimpleHash;
ths->destroy = destroySimpleHash;
ths->resize = resizeSimpleHash;
ths->hashFunc = simpleHashFunc;
}
contrib/test/traft/make_cluster/simpleHash.h 已删除 100644 → 0
浏览文件 @ 72ed99e8
#ifndef __SIMPLE_HASH_H__
#define __SIMPLE_HASH_H__
#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
uint32_t mySimpleHash(const char* data, size_t n, uint32_t seed);
typedef struct SimpleHashNode {
uint32_t hashCode;
void* key;
size_t keyLen;
void* data;
size_t dataLen;
struct SimpleHashNode* next;
} SimpleHashNode;
typedef struct SimpleHash {
// public:
int (*insert)(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen);
int (*remove)(struct SimpleHash* ths, char* key, size_t keyLen);
SimpleHashNode** (*find)(struct SimpleHash* ths, char* key, size_t keyLen);
// wrapper
int (*insert_cstr)(struct SimpleHash* ths, char* key, char* data);
int (*remove_cstr)(struct SimpleHash* ths, char* key);
SimpleHashNode** (*find_cstr)(struct SimpleHash* ths, char* key);
void (*print_cstr)(struct SimpleHash* ths);
void (*destroy)(struct SimpleHash* ths);
uint32_t length;
uint32_t elems;
// private:
void (*resize)(struct SimpleHash* ths);
uint32_t (*hashFunc)(const char* key, size_t keyLen);
SimpleHashNode** table;
} SimpleHash;
int insertCStrSimpleHash(struct SimpleHash* ths, char* key, char* data);
int removeCStrSimpleHash(struct SimpleHash* ths, char* key);
SimpleHashNode** findCStrSimpleHash(struct SimpleHash* ths, char* key);
void printCStrSimpleHash(struct SimpleHash* ths);
int insertSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen);
int removeSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen);
SimpleHashNode** findSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen);
void destroySimpleHash(struct SimpleHash* ths);
void resizeSimpleHash(struct SimpleHash* ths);
uint32_t simpleHashFunc(const char* key, size_t keyLen);
struct SimpleHash* newSimpleHash(size_t length);
#endif
contrib/test/traft/make_cluster/util.c 已删除 100644 → 0
浏览文件 @ 72ed99e8
#include "util.h"
#include <dirent.h>
#include <stdlib.h>
#include <string.h>
int dirOK(const char *path) {
DIR *dir = opendir(path);
if (dir != NULL) {
closedir(dir);
return 1;
} else {
return 0;
}
}
int splitString(const char *str, char *separator, char (*arr)[TOKEN_LEN], int n_arr) {
if (n_arr <= 0) {
return -1;
}
char *tmp = (char *)malloc(strlen(str) + 1);
strcpy(tmp, str);
char *context;
int n = 0;
char *token = strtok_r(tmp, separator, &context);
if (!token) {
goto ret;
}
strncpy(arr[n], token, TOKEN_LEN);
n++;
while (1) {
token = strtok_r(NULL, separator, &context);
if (!token || n >= n_arr) {
goto ret;
}
strncpy(arr[n], token, TOKEN_LEN);
n++;
}
ret:
free(tmp);
return n;
}
contrib/test/traft/make_cluster/util.h 已删除 100644 → 0
浏览文件 @ 72ed99e8
#ifndef TRAFT_UTIL_H
#define TRAFT_UTIL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "common.h"
int dirOK(const char *path);
int splitString(const char *str, char *separator, char (*arr)[TOKEN_LEN], int n_arr);
#ifdef __cplusplus
}
#endif
#endif
contrib/test/traft/rebalance_leader/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ 72ed99e8
add_executable(rebalanceLeader "")
target_sources(rebalanceLeader
PRIVATE
"raftMain.c"
"raftServer.c"
)
target_link_libraries(rebalanceLeader PUBLIC traft lz4 uv_a)
contrib/test/traft/rebalance_leader/common.h 已删除 100644 → 0
浏览文件 @ 72ed99e8
#ifndef TDENGINE_COMMON_H
#define TDENGINE_COMMON_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#define MAX_INSTANCE_NUM 100
#define MAX_PEERS 10
#define COMMAND_LEN 1024
#define TOKEN_LEN 128
#define DIR_LEN 256
#define HOST_LEN 64
#define ADDRESS_LEN (HOST_LEN + 16)
typedef struct {
char host[HOST_LEN];
uint32_t port;
} Addr;
typedef struct {
Addr me;
Addr peers[MAX_PEERS];
int peersCount;
char dir[DIR_LEN];
char dataDir[DIR_LEN + HOST_LEN * 2];
} SRaftServerConfig;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_COMMON_H
contrib/test/traft/rebalance_leader/raftMain.c 已删除 100644 → 0
浏览文件 @ 72ed99e8
#include <stdio.h>
#include <unistd.h>
#include <pthread.h>
#include <string.h>
#include <assert.h>
#include <getopt.h>
#include <time.h>
#include <stdlib.h>
#include <getopt.h>
#include <raft.h>
#include <raft/uv.h>
#include "raftServer.h"
#include "common.h"
const char *exe_name;
typedef struct LeaderState {
char address[48];
int leaderCount;
} LeaderState;
#define NODE_COUNT 3
LeaderState leaderStates[NODE_COUNT];
void printLeaderCount() {
for (int i = 0; i < NODE_COUNT; ++i) {
printf("%s: leaderCount:%d \n", leaderStates[i].address, leaderStates[i].leaderCount);
}
}
void updateLeaderStates(SRaftServer *pRaftServer) {
for (int i = 0; i < pRaftServer->instance[0].raft.configuration.n; ++i) {
snprintf(leaderStates[i].address, sizeof(leaderStates[i].address), "%s", pRaftServer->instance[0].raft.configuration.servers[i].address);
leaderStates[i].leaderCount = 0;
}
for (int i = 0; i < pRaftServer->instanceCount; ++i) {
struct raft *r = &pRaftServer->instance[i].raft;
char leaderAddress[128];
memset(leaderAddress, 0, sizeof(leaderAddress));
if (r->state == RAFT_LEADER) {
snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address);
} else if (r->state == RAFT_FOLLOWER) {
snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address);
}
for (int j = 0; j < NODE_COUNT; j++) {
if (strcmp(leaderAddress, leaderStates[j].address) == 0) {
leaderStates[j].leaderCount++;
}
}
}
}
void raftTransferCb(struct raft_transfer *req) {
SRaftServer *pRaftServer = req->data;
raft_free(req);
//printf("raftTransferCb: \n");
updateLeaderStates(pRaftServer);
//printLeaderCount();
int myLeaderCount;
for (int i = 0; i < NODE_COUNT; ++i) {
if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) {
myLeaderCount = leaderStates[i].leaderCount;
}
}
//printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT);
if (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) {
struct raft *r;
for (int j = 0; j < pRaftServer->instanceCount; ++j) {
if (pRaftServer->instance[j].raft.state == RAFT_LEADER) {
r = &pRaftServer->instance[j].raft;
break;
}
}
struct raft_transfer *transfer = raft_malloc(sizeof(*transfer));
transfer->data = pRaftServer;
uint64_t destRaftId;
int minIndex = -1;
int minLeaderCount = myLeaderCount;
for (int j = 0; j < NODE_COUNT; ++j) {
if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) {
continue;
}
if (leaderStates[j].leaderCount <= minLeaderCount) {
minLeaderCount = leaderStates[j].leaderCount;
minIndex = j;
}
}
char myHost[48];
uint16_t myPort;
uint16_t myVid;
decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid);
//printf("raftTransferCb transfer leader: vid[%u] choose: index:%d, leaderStates[%d].address:%s, leaderStates[%d].leaderCount:%d \n", minIndex, minIndex, leaderStates[minIndex].address, minIndex, leaderStates[minIndex].leaderCount);
char *destAddress = leaderStates[minIndex].address;
char tokens[MAX_PEERS][MAX_TOKEN_LEN];
splitString(destAddress, ":", tokens, 2);
char *destHost = tokens[0];
uint16_t destPort = atoi(tokens[1]);
destRaftId = encodeRaftId(destHost, destPort, myVid);
printf("\nraftTransferCb transfer leader: vgroupId:%u from:%s:%u --> to:%s:%u ", myVid, myHost, myPort, destHost, destPort);
fflush(stdout);
raft_transfer(r, transfer, destRaftId, raftTransferCb);
}
}
void parseAddr(const char *addr, char *host, int len, uint32_t *port) {
char* tmp = (char*)malloc(strlen(addr) + 1);
strcpy(tmp, addr);
char* context;
char* separator = ":";
char* token = strtok_r(tmp, separator, &context);
if (token) {
snprintf(host, len, "%s", token);
}
token = strtok_r(NULL, separator, &context);
if (token) {
sscanf(token, "%u", port);
}
free(tmp);
}
// only parse 3 tokens
int parseCommand3(const char* str, char* token1, char* token2, char* token3, int len)
{
char* tmp = (char*)malloc(strlen(str) + 1);
strcpy(tmp, str);
char* context;
char* separator = " ";
int n = 0;
char* token = strtok_r(tmp, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token1, token, len);
n++;
}
token = strtok_r(NULL, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token2, token, len);
n++;
}
token = strtok_r(NULL, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token3, token, len);
n++;
}
ret:
return n;
free(tmp);
}
// only parse 4 tokens
int parseCommand4(const char* str, char* token1, char* token2, char* token3, char *token4, int len)
{
char* tmp = (char*)malloc(strlen(str) + 1);
strcpy(tmp, str);
char* context;
char* separator = " ";
int n = 0;
char* token = strtok_r(tmp, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token1, token, len);
n++;
}
token = strtok_r(NULL, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token2, token, len);
n++;
}
token = strtok_r(NULL, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token3, token, len);
n++;
}
token = strtok_r(NULL, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token4, token, len);
n++;
}
ret:
return n;
free(tmp);
}
void *startServerFunc(void *param) {
SRaftServer *pServer = (SRaftServer*)param;
int32_t r = raftServerStart(pServer);
assert(r == 0);
return NULL;
}
// Console ---------------------------------
const char* state2String(unsigned short state) {
if (state == RAFT_UNAVAILABLE) {
return "RAFT_UNAVAILABLE";
} else if (state == RAFT_FOLLOWER) {
return "RAFT_FOLLOWER";
} else if (state == RAFT_CANDIDATE) {
return "RAFT_CANDIDATE";
} else if (state == RAFT_LEADER) {
return "RAFT_LEADER";
}
return "UNKNOWN_RAFT_STATE";
}
void printRaftState2(struct raft *r) {
char leaderAddress[128];
memset(leaderAddress, 0, sizeof(leaderAddress));
if (r->state == RAFT_LEADER) {
snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address);
} else if (r->state == RAFT_FOLLOWER) {
snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address);
}
for (int i = 0; i < r->configuration.n; ++i) {
char tmpAddress[128];
snprintf(tmpAddress, sizeof(tmpAddress), "%s", r->configuration.servers[i].address);
uint64_t raftId = r->configuration.servers[i].id;
char host[128];
uint16_t port;
uint16_t vid;
decodeRaftId(raftId, host, 128, &port, &vid);
char buf[512];
memset(buf, 0, sizeof(buf));
if (strcmp(tmpAddress, leaderAddress) == 0) {
snprintf(buf, sizeof(buf), "<%s:%u-%u-LEADER>\t", host, port, vid);
} else {
snprintf(buf, sizeof(buf), "<%s:%u-%u-FOLLOWER>\t", host, port, vid);
}
printf("%s", buf);
}
printf("\n");
}
void printRaftConfiguration(struct raft_configuration *c) {
printf("configuration: \n");
for (int i = 0; i < c->n; ++i) {
printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address);
}
}
void printRaftState(struct raft *r) {
printf("----Raft State: -----------\n");
printf("mem_addr: %p \n", r);
printf("my_id: %llu \n", r->id);
printf("address: %s \n", r->address);
printf("current_term: %llu \n", r->current_term);
printf("voted_for: %llu \n", r->voted_for);
printf("role: %s \n", state2String(r->state));
printf("commit_index: %llu \n", r->commit_index);
printf("last_applied: %llu \n", r->last_applied);
printf("last_stored: %llu \n", r->last_stored);
printf("configuration_index: %llu \n", r->configuration_index);
printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index);
printRaftConfiguration(&r->configuration);
printf("----------------------------\n");
}
void putValueCb(struct raft_apply *req, int status, void *result) {
raft_free(req);
struct raft *r = req->data;
if (status != 0) {
printf("putValueCb: %s \n", raft_errmsg(r));
} else {
printf("putValueCb: %s \n", "ok");
}
}
void putValue(struct raft *r, const char *value) {
struct raft_buffer buf;
buf.len = TOKEN_LEN;;
buf.base = raft_malloc(buf.len);
snprintf(buf.base, buf.len, "%s", value);
struct raft_apply *req = raft_malloc(sizeof(struct raft_apply));
req->data = r;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char*)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
void getValue(const char *key) {
char *ptr = getKV(key);
if (ptr) {
printf("get value: [%s] \n", ptr);
} else {
printf("value not found for key: [%s] \n", key);
}
}
void console(SRaftServer *pRaftServer) {
while (1) {
char cmd_buf[COMMAND_LEN];
memset(cmd_buf, 0, sizeof(cmd_buf));
printf("(console)> ");
char *ret = fgets(cmd_buf, COMMAND_LEN, stdin);
if (!ret) {
exit(-1);
}
int pos = strlen(cmd_buf);
if(cmd_buf[pos - 1] == '\n') {
cmd_buf[pos - 1] = '\0';
}
if (strncmp(cmd_buf, "", COMMAND_LEN) == 0) {
continue;
}
char cmd[TOKEN_LEN];
memset(cmd, 0, sizeof(cmd));
char param1[TOKEN_LEN];
memset(param1, 0, sizeof(param1));
char param2[TOKEN_LEN];
memset(param2, 0, sizeof(param2));
char param3[TOKEN_LEN];
memset(param2, 0, sizeof(param2));
parseCommand4(cmd_buf, cmd, param1, param2, param3, TOKEN_LEN);
if (strcmp(cmd, "addnode") == 0) {
printf("not support \n");
/*
char host[HOST_LEN];
uint32_t port;
parseAddr(param1, host, HOST_LEN, &port);
uint64_t rid = raftId(host, port);
struct raft_change *req = raft_malloc(sizeof(*req));
int r = raft_add(&pRaftServer->raft, req, rid, param1, NULL);
if (r != 0) {
printf("raft_add: %s \n", raft_errmsg(&pRaftServer->raft));
}
printf("add node: %lu %s \n", rid, param1);
struct raft_change *req2 = raft_malloc(sizeof(*req2));
r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, NULL);
if (r != 0) {
printf("raft_assign: %s \n", raft_errmsg(&pRaftServer->raft));
}
*/
} else if (strcmp(cmd, "dropnode") == 0) {
printf("not support \n");
} else if (strcmp(cmd, "quit") == 0 || strcmp(cmd, "exit") == 0) {
exit(0);
} else if (strcmp(cmd, "rebalance") == 0 && strcmp(param1, "leader") == 0) {
/*
updateLeaderStates(pRaftServer);
int myLeaderCount;
for (int i = 0; i < NODE_COUNT; ++i) {
if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) {
myLeaderCount = leaderStates[i].leaderCount;
}
}
while (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) {
printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT);
struct raft *r;
for (int j = 0; j < pRaftServer->instanceCount; ++j) {
if (pRaftServer->instance[j].raft.state == RAFT_LEADER) {
r = &pRaftServer->instance[j].raft;
}
}
struct raft_transfer *transfer = raft_malloc(sizeof(*transfer));
transfer->data = pRaftServer;
uint64_t destRaftId;
int minIndex = -1;
int minLeaderCount = myLeaderCount;
for (int j = 0; j < NODE_COUNT; ++j) {
if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) continue;
printf("-----leaderStates[%d].leaderCount:%d \n", j, leaderStates[j].leaderCount);
if (leaderStates[j].leaderCount <= minLeaderCount) {
minIndex = j;
printf("++++ assign minIndex : %d \n", minIndex);
}
}
printf("minIndex:%d minLeaderCount:%d \n", minIndex, minLeaderCount);
char myHost[48];
uint16_t myPort;
uint16_t myVid;
decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid);
char *destAddress = leaderStates[minIndex].address;
char tokens[MAX_PEERS][MAX_TOKEN_LEN];
splitString(destAddress, ":", tokens, 2);
char *destHost = tokens[0];
uint16_t destPort = atoi(tokens[1]);
destRaftId = encodeRaftId(destHost, destPort, myVid);
printf("destHost:%s destPort:%u myVid:%u", destHost, destPort, myVid);
raft_transfer(r, transfer, destRaftId, raftTransferCb);
sleep(1);
for (int i = 0; i < NODE_COUNT; ++i) {
if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) {
myLeaderCount = leaderStates[i].leaderCount;
}
}
}
*/
int leaderCount = 0;
struct raft *firstR;
for (int i = 0; i < pRaftServer->instanceCount; ++i) {
struct raft *r = &pRaftServer->instance[i].raft;
if (r->state == RAFT_LEADER) {
leaderCount++;
firstR = r;
}
}
if (leaderCount > pRaftServer->instanceCount / NODE_COUNT) {
struct raft_transfer *transfer = raft_malloc(sizeof(*transfer));
transfer->data = pRaftServer;
raft_transfer(firstR, transfer, 0, raftTransferCb);
}
} else if (strcmp(cmd, "put") == 0) {
char buf[256];
uint16_t vid;
sscanf(param1, "%hu", &vid);
snprintf(buf, sizeof(buf), "%s--%s", param2, param3);
putValue(&pRaftServer->instance[vid].raft, buf);
} else if (strcmp(cmd, "get") == 0) {
getValue(param1);
} else if (strcmp(cmd, "transfer") == 0) {
uint16_t vid;
sscanf(param1, "%hu", &vid);
struct raft_transfer transfer;
raft_transfer(&pRaftServer->instance[vid].raft, &transfer, 0, NULL);
} else if (strcmp(cmd, "state") == 0) {
for (int i = 0; i < pRaftServer->instanceCount; ++i) {
printf("instance %d: ", i);
printRaftState(&pRaftServer->instance[i].raft);
}
} else if (strcmp(cmd, "leader") == 0 && strcmp(param1, "state") == 0) {
updateLeaderStates(pRaftServer);
printf("\n--------------------------------------------\n");
printLeaderCount();
for (int i = 0; i < pRaftServer->instanceCount; ++i) {
printRaftState2(&pRaftServer->instance[i].raft);
}
printf("--------------------------------------------\n");
} else if (strcmp(cmd, "snapshot") == 0) {
printf("not support \n");
} else if (strcmp(cmd, "help") == 0) {
printf("addnode \"127.0.0.1:8888\" \n");
printf("dropnode \"127.0.0.1:8888\" \n");
printf("put key value \n");
printf("get key \n");
printf("state \n");
} else {
printf("unknown command: [%s], type \"help\" to see help \n", cmd);
}
//printf("cmd_buf: [%s] \n", cmd_buf);
}
}
void *startConsoleFunc(void *param) {
SRaftServer *pServer = (SRaftServer*)param;
console(pServer);
return NULL;
}
// Config ---------------------------------
void usage() {
printf("\nusage: \n");
printf("%s --me=127.0.0.1:10000 --dir=./data \n", exe_name);
printf("\n");
printf("%s --me=127.0.0.1:10000 --peers=127.0.0.1:10001,127.0.0.1:10002 --dir=./data \n", exe_name);
printf("%s --me=127.0.0.1:10001 --peers=127.0.0.1:10000,127.0.0.1:10002 --dir=./data \n", exe_name);
printf("%s --me=127.0.0.1:10002 --peers=127.0.0.1:10000,127.0.0.1:10001 --dir=./data \n", exe_name);
printf("\n");
}
void parseConf(int argc, char **argv, SRaftServerConfig *pConf) {
memset(pConf, 0, sizeof(*pConf));
int option_index, option_value;
option_index = 0;
static struct option long_options[] = {
{"help", no_argument, NULL, 'h'},
{"peers", required_argument, NULL, 'p'},
{"me", required_argument, NULL, 'm'},
{"dir", required_argument, NULL, 'd'},
{NULL, 0, NULL, 0}
};
while ((option_value = getopt_long(argc, argv, "hp:m:d:", long_options, &option_index)) != -1) {
switch (option_value) {
case 'm': {
parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port);
break;
}
case 'p': {
char tokens[MAX_PEERS][MAX_TOKEN_LEN];
int peerCount = splitString(optarg, ",", tokens, MAX_PEERS);
pConf->peersCount = peerCount;
for (int i = 0; i < peerCount; ++i) {
Addr *pAddr = &pConf->peers[i];
parseAddr(tokens[i], pAddr->host, sizeof(pAddr->host), &pAddr->port);
}
break;
}
case 'd': {
snprintf(pConf->dir, sizeof(pConf->dir), "%s", optarg);
break;
}
case 'h': {
usage();
exit(-1);
}
default: {
usage();
exit(-1);
}
}
}
snprintf(pConf->dataDir, sizeof(pConf->dataDir), "%s/%s_%u", pConf->dir, pConf->me.host, pConf->me.port);
}
void printConf(SRaftServerConfig *pConf) {
printf("\nconf: \n");
printf("me: %s:%u \n", pConf->me.host, pConf->me.port);
printf("peersCount: %d \n", pConf->peersCount);
for (int i = 0; i < pConf->peersCount; ++i) {
Addr *pAddr = &pConf->peers[i];
printf("peer%d: %s:%u \n", i, pAddr->host, pAddr->port);
}
printf("dataDir: %s \n\n", pConf->dataDir);
}
int main(int argc, char **argv) {
srand(time(NULL));
int32_t ret;
exe_name = argv[0];
if (argc < 3) {
usage();
exit(-1);
}
SRaftServerConfig conf;
parseConf(argc, argv, &conf);
printConf(&conf);
signal(SIGPIPE, SIG_IGN);
/*
char cmd_buf[COMMAND_LEN];
snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", conf.dataDir);
system(cmd_buf);
*/
struct raft_fsm fsm;
initFsm(&fsm);
SRaftServer raftServer;
ret = raftServerInit(&raftServer, &conf, &fsm);
assert(ret == 0);
pthread_t tidRaftServer;
pthread_create(&tidRaftServer, NULL, startServerFunc, &raftServer);
pthread_t tidConsole;
pthread_create(&tidConsole, NULL, startConsoleFunc, &raftServer);
while (1) {
sleep(10);
}
return 0;
}
contrib/test/traft/rebalance_leader/raftServer.c 已删除 100644 → 0
浏览文件 @ 72ed99e8
#include <unistd.h>
#include <stdlib.h>
#include "common.h"
#include "raftServer.h"
//char *keys = malloc(MAX_RECORD_COUNT * MAX_KV_LEN);;
//char *values = malloc(MAX_RECORD_COUNT * MAX_KV_LEN);
char keys[MAX_KV_LEN][MAX_RECORD_COUNT];
char values[MAX_KV_LEN][MAX_RECORD_COUNT];
int writeIndex = 0;
void initStore() {
}
void destroyStore() {
//free(keys);
//free(values);
}
void putKV(const char *key, const char *value) {
if (writeIndex < MAX_RECORD_COUNT) {
strncpy(keys[writeIndex], key, MAX_KV_LEN);
strncpy(values[writeIndex], value, MAX_KV_LEN);
writeIndex++;
}
}
char *getKV(const char *key) {
for (int i = 0; i < MAX_RECORD_COUNT; ++i) {
if (strcmp(keys[i], key) == 0) {
return values[i];
}
}
return NULL;
}
int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr)
{
if (n_arr <= 0) {
return -1;
}
char* tmp = (char*)malloc(strlen(str) + 1);
strcpy(tmp, str);
char* context;
int n = 0;
char* token = strtok_r(tmp, separator, &context);
if (!token) {
goto ret;
}
strncpy(arr[n], token, MAX_TOKEN_LEN);
n++;
while (1) {
token = strtok_r(NULL, separator, &context);
if (!token || n >= n_arr) {
goto ret;
}
strncpy(arr[n], token, MAX_TOKEN_LEN);
n++;
}
ret:
free(tmp);
return n;
}
/*
uint64_t raftId(const char *host, uint32_t port) {
uint32_t host_uint32 = (uint32_t)inet_addr(host);
assert(host_uint32 != (uint32_t)-1);
uint64_t code = ((uint64_t)host_uint32) << 32 | port;
return code;
}
*/
/*
uint64_t encodeRaftId(const char *host, uint16_t port, uint16_t vid) {
uint64_t raftId;
uint32_t host_uint32 = (uint32_t)inet_addr(host);
assert(host_uint32 != (uint32_t)-1);
raftId = (((uint64_t)host_uint32) << 32) | (((uint32_t)port) << 16) | vid;
return raftId;
}
void decodeRaftId(uint64_t raftId, char *host, int32_t len, uint16_t *port, uint16_t *vid) {
uint32_t host32 = (uint32_t)((raftId >> 32) & 0x00000000FFFFFFFF);
struct in_addr addr;
addr.s_addr = host32;
snprintf(host, len, "%s", inet_ntoa(addr));
*port = (uint16_t)((raftId & 0x00000000FFFF0000) >> 16);
*vid = (uint16_t)(raftId & 0x000000000000FFFF);
}
*/
int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm) {
int ret;
snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host);
pRaftServer->port = pConf->me.port;
snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port);
//strncpy(pRaftServer->dir, pConf->dataDir, sizeof(pRaftServer->dir));
ret = uv_loop_init(&pRaftServer->loop);
if (ret != 0) {
fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret));
assert(0);
}
ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop);
if (ret != 0) {
fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret);
assert(0);
}
uint16_t vid;
pRaftServer->instanceCount = 20;
for (int i = 0; i < pRaftServer->instanceCount; ++i)
{
//vid = 0;
vid = i;
pRaftServer->instance[vid].raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid);
snprintf(pRaftServer->instance[vid].dir, sizeof(pRaftServer->instance[vid].dir), "%s_%llu", pConf->dataDir, pRaftServer->instance[vid].raftId);
char cmd_buf[COMMAND_LEN];
snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", pRaftServer->instance[vid].dir);
system(cmd_buf);
sleep(1);
pRaftServer->instance[vid].fsm = pFsm;
ret = raft_uv_init(&pRaftServer->instance[vid].io, &pRaftServer->loop, pRaftServer->instance[vid].dir, &pRaftServer->transport);
if (ret != 0) {
fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft));
assert(0);
}
ret = raft_init(&pRaftServer->instance[vid].raft, &pRaftServer->instance[vid].io, pRaftServer->instance[vid].fsm, pRaftServer->instance[vid].raftId, pRaftServer->address);
if (ret != 0) {
fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft));
assert(0);
}
struct raft_configuration conf;
raft_configuration_init(&conf);
raft_configuration_add(&conf, pRaftServer->instance[vid].raftId, pRaftServer->address, RAFT_VOTER);
printf("add myself: %llu - %s \n", pRaftServer->instance[vid].raftId, pRaftServer->address);
for (int i = 0; i < pConf->peersCount; ++i) {
const Addr *pAddr = &pConf->peers[i];
raft_id rid = encodeRaftId(pAddr->host, pAddr->port, vid);
char addrBuf[ADDRESS_LEN];
snprintf(addrBuf, sizeof(addrBuf), "%s:%u", pAddr->host, pAddr->port);
raft_configuration_add(&conf, rid, addrBuf, RAFT_VOTER);
printf("add peers: %llu - %s \n", rid, addrBuf);
}
raft_bootstrap(&pRaftServer->instance[vid].raft, &conf);
}
return 0;
}
int32_t raftServerStart(SRaftServer *pRaftServer) {
int ret;
for (int i = 0; i < pRaftServer->instanceCount; ++i) {
ret = raft_start(&pRaftServer->instance[i].raft);
if (ret != 0) {
fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[i].raft));
}
}
uv_run(&pRaftServer->loop, UV_RUN_DEFAULT);
}
void raftServerClose(SRaftServer *pRaftServer) {
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) {
char *msg = (char*)buf->base;
printf("fsm apply: %s \n", msg);
char arr[2][MAX_TOKEN_LEN];
splitString(msg, "--", arr, 2);
putKV(arr[0], arr[1]);
return 0;
}
int32_t initFsm(struct raft_fsm *fsm) {
initStore();
fsm->apply = fsmApplyCb;
return 0;
}
contrib/test/traft/rebalance_leader/raftServer.h 已删除 100644 → 0
浏览文件 @ 72ed99e8
#ifndef TDENGINE_RAFT_SERVER_H
#define TDENGINE_RAFT_SERVER_H
#ifdef __cplusplus
extern "C" {
#endif
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <string.h>
#include "raft.h"
#include "raft/uv.h"
#include "common.h"
// simulate a db store, just for test
#define MAX_KV_LEN 20
#define MAX_RECORD_COUNT 16
//char *keys;
//char *values;
//int writeIndex;
void initStore();
void destroyStore();
void putKV(const char *key, const char *value);
char *getKV(const char *key);
typedef struct {
char dir[DIR_LEN + HOST_LEN * 4]; /* Data dir of UV I/O backend */
raft_id raftId; /* For vote */
struct raft_fsm *fsm; /* Sample application FSM */
struct raft raft; /* Raft instance */
struct raft_io io; /* UV I/O backend */
} SInstance;
typedef struct {
char host[HOST_LEN];
uint32_t port;
char address[ADDRESS_LEN]; /* Raft instance address */
struct uv_loop_s loop; /* UV loop */
struct raft_uv_transport transport; /* UV I/O backend transport */
SInstance instance[MAX_INSTANCE_NUM];
int32_t instanceCount;
} SRaftServer;
#define MAX_TOKEN_LEN 32
int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr);
int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm);
int32_t raftServerStart(SRaftServer *pRaftServer);
void raftServerClose(SRaftServer *pRaftServer);
int initFsm(struct raft_fsm *fsm);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_RAFT_SERVER_H
contrib/test/traft/single_node/CMakeLists.txt 0 → 100644
浏览文件 @ 6e358fd4
add_executable(singleNode "")
target_sources(singleNode
PRIVATE
"singleNode.c"
)
target_link_libraries(singleNode PUBLIC traft lz4 uv_a)
contrib/test/traft/single_node/clear.sh 0 → 100644
浏览文件 @ 6e358fd4
#!/bin/bash
rm -rf 127.0.0.1*
rm -rf ./data
contrib/test/traft/single_node/cmd 0 → 100644
浏览文件 @ 6e358fd4
all:
gcc singleNode.c -I ../../include/ ../../.libs/libraft.a -o singleNode -luv -llz4 -lpthread -g
clean:
rm -f singleNode
sh clear.sh
contrib/test/traft/single_node/singleNode.c 0 → 100644
浏览文件 @ 6e358fd4
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
//#include <uv.h>
#include "raft.h"
SRaftEnv raftEnv;
typedef struct Tsdb {
uint64_t lastApplyIndex;
void *mem;
void *imm;
void *store;
} Tsdb;
void tsdbWrite(Tsdb *t, char *msg) {}
void *startFunc(void *param) {
SRaftEnv *pSRaftEnv = (SRaftEnv *)param;
int32_t r = raftEnvStart(pSRaftEnv);
assert(r == 0);
return NULL;
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) {
// get commit value
char *msg = (char *)buf->base;
printf("fsm apply: index:%llu value:%s \n", index, msg);
Tsdb *t = pFsm->data;
if (index > t->lastApplyIndex) {
// apply value into tsdb
tsdbWrite(t, msg);
// update lastApplyIndex
t->lastApplyIndex = index;
}
return 0;
}
void putValueCb(struct raft_apply *req, int status, void *result) {
void *ptr = req->data;
if (status != 0) {
printf("putValueCb error \n");
} else {
printf("putValueCb ok \n");
}
free(ptr);
free(req);
}
void submitValue() {
// prepare value
struct raft_buffer buf;
buf.len = 32;
void *ptr = malloc(buf.len);
buf.base = ptr;
snprintf(buf.base, buf.len, "%ld", time(NULL));
// get raft
struct raft *r = getRaft(&raftEnv, 100);
assert(r != NULL);
// printRaftState(r);
// submit value
struct raft_apply *req = malloc(sizeof(*req));
req->data = ptr;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char *)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
int main(int argc, char **argv) {
// init raft env
int r = raftEnvInit(&raftEnv, "127.0.0.1", 38000, "./data");
assert(r == 0);
// start raft env
pthread_t tid;
pthread_create(&tid, NULL, startFunc, &raftEnv);
// wait for start, just for simple
while (raftEnv.isStart != 1) {
sleep(1);
}
// init fsm
struct raft_fsm *pFsm = malloc(sizeof(*pFsm));
pFsm->apply = fsmApplyCb;
Tsdb *tsdb = malloc(sizeof(*tsdb));
pFsm->data = tsdb;
// add vgroup, id = 100, only has one replica
r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm);
assert(r == 0);
// for test: submit a value every second
while (1) {
sleep(1);
submitValue();
}
return 0;
}
include/common/tmsg.h
浏览文件 @ 6e358fd4
......@@ -1529,17 +1529,22 @@ typedef struct SMqSetCVgReq {
} SMqSetCVgReq;
static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) {
int32_t tlen = sizeof(SSubQueryMsg) + pMsg->contentLen;
if (buf == NULL) return tlen;
memcpy(*buf, pMsg, tlen);
*buf = POINTER_SHIFT(*buf, tlen);
int32_t tlen = 0;
tlen += taosEncodeFixedU64(buf, pMsg->sId);
tlen += taosEncodeFixedU64(buf, pMsg->queryId);
tlen += taosEncodeFixedU64(buf, pMsg->taskId);
tlen += taosEncodeFixedU32(buf, pMsg->contentLen);
tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen);
return tlen;
}
static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) {
int32_t tlen = sizeof(SSubQueryMsg) + ((SSubQueryMsg*)buf)->contentLen;
memcpy(pMsg, buf, tlen);
return POINTER_SHIFT(buf, tlen);
buf = taosDecodeFixedU64(buf, &pMsg->sId);
buf = taosDecodeFixedU64(buf, &pMsg->queryId);
buf = taosDecodeFixedU64(buf, &pMsg->taskId);
buf = taosDecodeFixedU32(buf, &pMsg->contentLen);
buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen);
return buf;
}
static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) {
......@@ -1574,32 +1579,47 @@ typedef struct SMqSetCVgRsp {
char cGroup[TSDB_CONSUMER_GROUP_LEN];
} SMqSetCVgRsp;
typedef struct SMqCVConsumeReq {
typedef struct SMqConsumeReq {
int64_t reqId;
int64_t offset;
int64_t consumerId;
int64_t blockingTime;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN];
} SMqCVConsumeReq;
} SMqConsumeReq;
typedef struct SMqConsumeRspBlock {
int32_t bodyLen;
char topicName[TSDB_TOPIC_FNAME_LEN];
char body[];
} SMqConsumeRspBlock;
typedef struct SMqColData {
int16_t colId;
int16_t type;
int16_t bytes;
char data[];
} SMqColData;
typedef struct SMqCVConsumeRsp {
int64_t reqId;
int64_t clientId;
typedef struct SMqTbData {
int64_t uid;
int32_t numOfCols;
int32_t numOfRows;
SMqColData colData[];
} SMqTbData;
typedef struct SMqTopicBlk {
char topicName[TSDB_TOPIC_FNAME_LEN];
int64_t committedOffset;
int64_t receiveOffset;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
int32_t bodyLen;
char topicName[TSDB_TOPIC_FNAME_LEN];
SMqConsumeRspBlock blocks[];
} SMqCvConsumeRsp;
int32_t numOfTb;
SMqTbData tbData[];
} SMqTopicData;
typedef struct SMqConsumeRsp {
int64_t reqId;
int64_t clientId;
int32_t bodyLen;
int32_t numOfTopics;
SMqTopicData data[];
} SMqConsumeRsp;
#ifdef __cplusplus
}
......
include/libs/executor/executor.h
浏览文件 @ 6e358fd4
......@@ -26,17 +26,26 @@ typedef void* qTaskInfo_t;
typedef void* DataSinkHandle;
struct SSubplan;
/**
* Create the exec task for streaming mode
* @param pMsg
* @param streamReadHandle
* @return
*/
qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg *pMsg, void* streamReadHandle);
int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input);
/**
* Create the exec task object according to task json
* @param tsdb
* @param readHandle
* @param vgId
* @param pTaskInfoMsg
* @param pTaskInfo
* @param qId
* @return
*/
int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
int32_t qCreateExecTask(void* readHandle, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
/**
* The main task execution function, including query on both table and multiple tables,
......@@ -53,63 +62,63 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds);
* this function will be blocked to wait for the query execution completed or paused,
* in which case enough results have been produced already.
*
* @param qinfo
* @param tinfo
* @return
*/
int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext);
int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext);
/**
*
* Retrieve the actual results to fill the response message payload.
* Note that this function must be executed after qRetrieveQueryResultInfo is invoked.
*
* @param qinfo qinfo object
* @param tinfo tinfo object
* @param pRsp response message
* @param contLen payload length
* @return
*/
//int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
/**
* return the transporter context (RPC)
* @param qinfo
* @param tinfo
* @return
*/
void* qGetResultRetrieveMsg(qTaskInfo_t qinfo);
void* qGetResultRetrieveMsg(qTaskInfo_t tinfo);
/**
* kill the ongoing query and free the query handle and corresponding resources automatically
* @param qinfo qhandle
* @param tinfo qhandle
* @return
*/
int32_t qKillTask(qTaskInfo_t qinfo);
int32_t qKillTask(qTaskInfo_t tinfo);
/**
* kill the ongoing query asynchronously
* @param qinfo qhandle
* @param tinfo qhandle
* @return
*/
int32_t qAsyncKillTask(qTaskInfo_t qinfo);
int32_t qAsyncKillTask(qTaskInfo_t tinfo);
/**
* return whether query is completed or not
* @param qinfo
* @param tinfo
* @return
*/
int32_t qIsTaskCompleted(qTaskInfo_t qinfo);
int32_t qIsTaskCompleted(qTaskInfo_t tinfo);
/**
* destroy query info structure
* @param qHandle
*/
void qDestroyTask(qTaskInfo_t qHandle);
void qDestroyTask(qTaskInfo_t tinfo);
/**
* Get the queried table uid
* @param qHandle
* @return
*/
int64_t qGetQueriedTableUid(qTaskInfo_t qHandle);
int64_t qGetQueriedTableUid(qTaskInfo_t tinfo);
/**
* Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks.
......@@ -136,7 +145,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t
* @param type operation type: ADD|DROP
* @return
*/
int32_t qUpdateQueriedTableIdList(qTaskInfo_t qinfo, int64_t uid, int32_t type);
int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
//================================================================================================
// query handle management
......
include/libs/planner/planner.h
浏览文件 @ 6e358fd4
......@@ -178,7 +178,7 @@ struct SQueryNode;
* @param requestId
* @return
*/
int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId);
int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId);
// Set datasource of this subplan, multiple calls may be made to a subplan.
// @subplan subplan to be schedule
......
include/util/compare.h
浏览文件 @ 6e358fd4
......@@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_COMPARE_H
#define _TD_UTIL_COMPARE_H
#ifndef _TD_UTIL_COMPARE_H_
#define _TD_UTIL_COMPARE_H_
#ifdef __cplusplus
extern "C" {
......@@ -35,67 +35,65 @@ extern "C" {
#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y)))
#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y)))
#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_' }
#define PATTERN_COMPARE_INFO_INITIALIZER \
{ '%', '_' }
typedef struct SPatternCompareInfo {
char matchAll; // symbol for match all wildcard, default: '%'
char matchOne; // symbol for match one wildcard, default: '_'
} SPatternCompareInfo;
int patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t taosArrayCompareString(const void* a, const void* b);
int32_t taosArrayCompareString(const void *a, const void *b);
int32_t setCompareBytes1(const void *pLeft, const void *pRight);
int32_t setCompareBytes2(const void *pLeft, const void *pRight);
int32_t setCompareBytes4(const void *pLeft, const void *pRight);
int32_t setCompareBytes8(const void *pLeft, const void *pRight);
int32_t compareInt8Val(const void *pLeft, const void *pRight);
int32_t compareInt16Val(const void *pLeft, const void *pRight);
int32_t compareInt32Val(const void *pLeft, const void *pRight);
int32_t compareInt64Val(const void *pLeft, const void *pRight);
int32_t compareInt16Val(const void *pLeft, const void *pRight);
int32_t compareInt8Val(const void *pLeft, const void *pRight);
int32_t compareUint8Val(const void *pLeft, const void *pRight);
int32_t compareUint16Val(const void *pLeft, const void *pRight);
int32_t compareUint32Val(const void *pLeft, const void *pRight);
int32_t compareUint64Val(const void *pLeft, const void *pRight);
int32_t compareUint16Val(const void *pLeft, const void *pRight);
int32_t compareUint8Val(const void* pLeft, const void* pRight);
int32_t compareFloatVal(const void *pLeft, const void *pRight);
int32_t compareDoubleVal(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight);
int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight);
int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
int32_t compareStrRegexComp(const void *pLeft, const void *pRight);
int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight);
int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight);
int32_t compareFindItemInSet(const void *pLeft, const void *pRight);
int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
int32_t compareInt16ValDesc(const void* pLeft, const void* pRight);
int32_t compareInt32ValDesc(const void* pLeft, const void* pRight);
int32_t compareInt64ValDesc(const void* pLeft, const void* pRight);
int32_t compareFloatValDesc(const void* pLeft, const void* pRight);
int32_t compareDoubleValDesc(const void* pLeft, const void* pRight);
int32_t compareUint8ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint16ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint32ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint64ValDesc(const void* pLeft, const void* pRight);
int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight);
int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight);
int32_t compareInt16ValDesc(const void *pLeft, const void *pRight);
int32_t compareInt32ValDesc(const void *pLeft, const void *pRight);
int32_t compareInt64ValDesc(const void *pLeft, const void *pRight);
int32_t compareFloatValDesc(const void *pLeft, const void *pRight);
int32_t compareDoubleValDesc(const void *pLeft, const void *pRight);
int32_t compareUint8ValDesc(const void *pLeft, const void *pRight);
int32_t compareUint16ValDesc(const void *pLeft, const void *pRight);
int32_t compareUint32ValDesc(const void *pLeft, const void *pRight);
int32_t compareUint64ValDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_COMPARE_H*/
#endif /*_TD_UTIL_COMPARE_H_*/
include/util/tcoding.h
浏览文件 @ 6e358fd4
......@@ -372,7 +372,7 @@ static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) {
}
// ---- binary
static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int valueLen) {
static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t valueLen) {
int tlen = 0;
if (buf != NULL) {
......@@ -384,14 +384,19 @@ static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int valu
return tlen;
}
static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int valueLen) {
uint64_t size = 0;
static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int32_t valueLen) {
*value = malloc((size_t)valueLen);
if (*value == NULL) return NULL;
memcpy(*value, buf, (size_t)size);
memcpy(*value, buf, (size_t)valueLen);
return POINTER_SHIFT(buf, size);
return POINTER_SHIFT(buf, valueLen);
}
static FORCE_INLINE void *taosDecodeBinaryTo(void *buf, void *value, int32_t valueLen) {
memcpy(value, buf, (size_t)valueLen);
return POINTER_SHIFT(buf, valueLen);
}
#endif
......
source/client/inc/clientInt.h
浏览文件 @ 6e358fd4
......@@ -136,6 +136,7 @@ typedef struct SReqResultInfo {
TAOS_ROW row;
char **pCol;
uint32_t numOfRows;
uint64_t totalRows;
uint32_t current;
bool completed;
} SReqResultInfo;
......
source/client/src/clientEnv.c
浏览文件 @ 6e358fd4
......@@ -110,7 +110,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) {
rpcInit.user = (char *)user;
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.ckey = "key";
// rpcInit.spi = 1;
rpcInit.spi = 1;
rpcInit.secret = (char *)auth;
void* pDnodeConn = rpcOpen(&rpcInit);
......@@ -196,6 +196,10 @@ static void doDestroyRequest(void* p) {
doFreeReqResultInfo(&pRequest->body.resInfo);
qDestroyQueryDag(pRequest->body.pDag);
if (pRequest->body.showInfo.pArray != NULL) {
taosArrayDestroy(pRequest->body.showInfo.pArray);
}
deregisterRequest(pRequest);
tfree(pRequest);
}
......
source/client/src/clientImpl.c
浏览文件 @ 6e358fd4
......@@ -25,9 +25,9 @@
static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest);
static void destroySendMsgInfo(SMsgSendInfo* pMsgBody);
static void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
static bool stringLengthCheck(const char* str, size_t maxsize) {
static bool stringLengthCheck(const char* str, size_t maxsize) {
if (str == NULL) {
return false;
}
......@@ -59,7 +59,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
}
static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo);
static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema);
static void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) {
if (taos_init() != TSDB_CODE_SUCCESS) {
......@@ -117,7 +117,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo));
p->mgmtEp = epSet;
p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
p->pAppHbMgr = appHbMgrInit(p);
/*p->pAppHbMgr = appHbMgrInit(p);*/
taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
pInst = &p;
......@@ -202,43 +202,38 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
return TSDB_CODE_SUCCESS;
}
int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) {
int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, SArray* pNodeList) {
pRequest->type = pQueryNode->type;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
int32_t code = qCreateQueryDag(pQueryNode, pDag, pRequest->requestId);
SSchema* pSchema = NULL;
int32_t numOfCols = 0;
int32_t code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &numOfCols, pNodeList, pRequest->requestId);
if (code != 0) {
return code;
}
if (pQueryNode->type == TSDB_SQL_SELECT) {
SArray* pa = taosArrayGetP((*pDag)->pSubplans, 0);
SSubplan* pPlan = taosArrayGetP(pa, 0);
SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema);
setResSchemaInfo(pResInfo, pDataBlockSchema);
setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols);
pRequest->type = TDMT_VND_QUERY;
}
return code;
}
void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema) {
assert(pDataBlockSchema != NULL && pDataBlockSchema->numOfCols > 0);
void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) {
assert(pSchema != NULL && numOfCols > 0);
pResInfo->numOfCols = pDataBlockSchema->numOfCols;
pResInfo->fields = calloc(pDataBlockSchema->numOfCols, sizeof(pDataBlockSchema->pSchema[0]));
pResInfo->numOfCols = numOfCols;
pResInfo->fields = calloc(numOfCols, sizeof(pSchema[0]));
for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
SSchema* pSchema = &pDataBlockSchema->pSchema[i];
pResInfo->fields[i].bytes = pSchema->bytes;
pResInfo->fields[i].type = pSchema->type;
tstrncpy(pResInfo->fields[i].name, pSchema->name, tListLen(pResInfo->fields[i].name));
pResInfo->fields[i].bytes = pSchema[i].bytes;
pResInfo->fields[i].type = pSchema[i].type;
tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name));
}
}
int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) {
int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) {
if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
......@@ -256,14 +251,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) {
return pRequest->code;
}
SArray *execNode = taosArrayInit(4, sizeof(SQueryNodeAddr));
SQueryNodeAddr addr = {.numOfEps = 1, .inUse = 0, .nodeId = 2};
addr.epAddr[0].port = 7100;
strcpy(addr.epAddr[0].fqdn, "localhost");
taosArrayPush(execNode, &addr);
return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, execNode, pDag, &pRequest->body.pQueryJob);
return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob);
}
typedef struct tmq_t tmq_t;
......@@ -399,7 +387,9 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql,
// todo check for invalid sql statement and return with error code
CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, pRequest->requestId), _return);
SSchema *schema = NULL;
int32_t numOfCols = 0;
CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), _return);
pStr = qDagToString(pRequest->body.pDag);
if(pStr == NULL) {
......@@ -492,6 +482,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
SRequestObj *pRequest = NULL;
SQueryNode *pQueryNode = NULL;
SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
......@@ -500,8 +491,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
if (qIsDdlQuery(pQueryNode)) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return);
} else {
CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag), _return);
CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return);
pRequest->code = terrno;
}
......@@ -571,7 +562,7 @@ STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uin
tsem_wait(&pRequest->body.rspSem);
if (pRequest->code != TSDB_CODE_SUCCESS) {
const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno);
const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code);
printf("failed to connect to server, reason: %s\n\n", errorMsg);
destroyRequest(pRequest);
......@@ -719,13 +710,17 @@ void* doFetchRow(SRequestObj* pRequest) {
return NULL;
}
int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData);
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pResInfo->pData);
if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code;
return NULL;
}
setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData);
setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData);
tscDebug("0x%"PRIx64 " fetch results, numOfRows:%d total Rows:%"PRId64", complete:%d, reqId:0x%"PRIx64, pRequest->self, pResInfo->numOfRows,
pResInfo->totalRows, pResInfo->completed, pRequest->requestId);
if (pResultInfo->numOfRows == 0) {
return NULL;
}
......@@ -855,7 +850,7 @@ void setConnectionDB(STscObj* pTscObj, const char* db) {
pthread_mutex_unlock(&pTscObj->mutex);
}
void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
assert(pResultInfo != NULL && pRsp != NULL);
pResultInfo->pRspMsg = (const char*) pRsp;
......@@ -864,5 +859,6 @@ void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* p
pResultInfo->current = 0;
pResultInfo->completed = (pRsp->completed == 1);
pResultInfo->totalRows += pResultInfo->numOfRows;
setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows);
}
source/client/src/clientMsgHandler.c
浏览文件 @ 6e358fd4
......@@ -72,7 +72,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY};
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);
/*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);*/
// pRequest->body.resInfo.pRspMsg = pMsg->pData;
tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId,
......@@ -145,6 +145,12 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) {
}
pSchema = pMetaMsg->pSchema;
tfree(pRequest->body.resInfo.pRspMsg);
pRequest->body.resInfo.pRspMsg = pMsg->pData;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
if (pResInfo->fields == NULL) {
TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD));
for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) {
tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name));
......@@ -152,12 +158,10 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) {
pFields[i].bytes = pSchema[i].bytes;
}
pRequest->body.resInfo.pRspMsg = pMsg->pData;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
pResInfo->fields = pFields;
pResInfo->numOfCols = pMetaMsg->numOfColumns;
}
pResInfo->numOfCols = pMetaMsg->numOfColumns;
pRequest->body.showInfo.execId = pShow->showId;
// todo
......
source/client/test/clientTests.cpp
浏览文件 @ 6e358fd4
......@@ -17,8 +17,9 @@
#include <taoserror.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
......@@ -451,39 +452,39 @@ TEST(testCase, driverInit_Test) {
//
// taos_close(pConn);
//}
//
//TEST(testCase, show_table_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "show tables");
// if (taos_errno(pRes) != 0) {
// printf("failed to show tables, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// }
//
// pRes = taos_query(pConn, "show abc1.tables");
// if (taos_errno(pRes) != 0) {
// printf("failed to show tables, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// int32_t count = 0;
// char str[512] = {0};
//
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%d: %s\n", ++count, str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
TEST(testCase, show_table_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "show tables");
if (taos_errno(pRes) != 0) {
printf("failed to show tables, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
}
pRes = taos_query(pConn, "show abc1.tables");
if (taos_errno(pRes) != 0) {
printf("failed to show tables, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
int32_t count = 0;
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%d: %s\n", ++count, str);
}
taos_free_result(pRes);
taos_close(pConn);
}
//TEST(testCase, drop_stable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
......@@ -525,99 +526,99 @@ TEST(testCase, driverInit_Test) {
// taosHashCleanup(phash);
//}
//
TEST(testCase, create_topic_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == nullptr);
int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0);
taos_free_result(pRes);
char* sql = "select * from tu";
pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql));
taos_free_result(pRes);
taos_close(pConn);
}
//TEST(testCase, insert_test) {
//TEST(testCase, create_topic_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// ASSERT_NE(pConn, nullptr);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in use db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "insert into t_2 values(now, 1)");
// if (taos_errno(pRes) != 0) {
// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// ASSERT_TRUE(pFields == nullptr);
//
// int32_t numOfFields = taos_num_fields(pRes);
// ASSERT_EQ(numOfFields, 0);
//
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// char* sql = "select * from tu";
// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql));
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, projection_query_tables) {
//TEST(testCase, insert_test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// ASSERT_NE(pConn, nullptr);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// taos_free_result(pRes);
//
//// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
//// if (taos_errno(pRes) != 0) {
//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
//// }
//// taos_free_result(pRes);
////
//// pRes = taos_query(pConn, "create table tu using st1 tags(1)");
//// if (taos_errno(pRes) != 0) {
//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
//// }
//// taos_free_result(pRes);
////
//// for(int32_t i = 0; i < 100; ++i) {
//// char sql[512] = {0};
//// sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
//// TAOS_RES* p = taos_query(pConn, sql);
//// if (taos_errno(p) != 0) {
//// printf("failed to insert data, reason:%s\n", taos_errstr(p));
//// }
////
//// taos_free_result(p);
//// }
//
// pRes = taos_query(pConn, "select * from tu");
// pRes = taos_query(pConn, "insert into t_2 values(now, 1)");
// if (taos_errno(pRes) != 0) {
// printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
TEST(testCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table tu using st1 tags(1)");
if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
for(int32_t i = 0; i < 100000; ++i) {
char sql[512] = {0};
sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
TAOS_RES* p = taos_query(pConn, sql);
if (taos_errno(p) != 0) {
printf("failed to insert data, reason:%s\n", taos_errstr(p));
}
taos_free_result(p);
}
pRes = taos_query(pConn, "select * from tu");
if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
}
//TEST(testCase, projection_query_stables) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// ASSERT_NE(pConn, nullptr);
......@@ -679,3 +680,5 @@ TEST(testCase, create_topic_Test) {
// taos_free_result(pRes);
// taos_close(pConn);
//}
#pragma GCC diagnostic pop
\ No newline at end of file
source/common/src/tglobal.c
浏览文件 @ 6e358fd4
......@@ -44,7 +44,7 @@ int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds;
int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default
int32_t tsMaxShellConns = 50000;
int32_t tsMaxConnections = 5000;
int32_t tsMaxConnections = 50000;
int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 4;
......
source/common/src/ttime.c
浏览文件 @ 6e358fd4
......@@ -100,12 +100,12 @@ int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t t
} else if (checkTzPresent(timestr, len)) {
return parseTimeWithTz(timestr, time, timePrec, 0);
} else {
return (*parseLocaltimeFp[day_light])(timestr, time, timePrec);
return (*parseLocaltimeFp[day_light])((char*)timestr, time, timePrec);
}
}
bool checkTzPresent(const char *str, int32_t len) {
char *seg = forwardToTimeStringEnd(str);
bool checkTzPresent(const char* str, int32_t len) {
char* seg = forwardToTimeStringEnd((char*)str);
int32_t seg_len = len - (int32_t)(seg - str);
char *c = &seg[seg_len - 1];
......@@ -267,7 +267,7 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
#endif
int64_t fraction = 0;
str = forwardToTimeStringEnd(timestr);
str = forwardToTimeStringEnd((char*)timestr);
if ((str[0] == 'Z' || str[0] == 'z') && str[1] == '\0') {
/* utc time, no millisecond, return directly*/
......
source/common/test/commonTests.cpp
浏览文件 @ 6e358fd4
#include <gtest/gtest.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
......@@ -94,3 +95,5 @@ TEST(testCase, toInteger_test) {
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, -1);
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/dnode/mgmt/impl/src/dndMnode.c
浏览文件 @ 6e358fd4
......@@ -621,5 +621,7 @@ int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *enc
int32_t code = mndRetriveAuth(pMnode, user, spi, encrypt, secret, ckey);
dndReleaseMnode(pDnode, pMnode);
dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt);
return code;
}
source/dnode/mgmt/impl/src/dndTransport.c
浏览文件 @ 6e358fd4
......@@ -171,7 +171,7 @@ static int32_t dndInitClient(SDnode *pDnode) {
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.label = "DND-C";
rpcInit.label = "D-C";
rpcInit.numOfThreads = 1;
rpcInit.cfp = dndProcessResponse;
rpcInit.sessions = 1024;
......@@ -179,9 +179,13 @@ static int32_t dndInitClient(SDnode *pDnode) {
rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000;
rpcInit.user = INTERNAL_USER;
rpcInit.ckey = INTERNAL_CKEY;
rpcInit.secret = INTERNAL_SECRET;
rpcInit.spi = 1;
rpcInit.parent = pDnode;
char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
rpcInit.secret = pass;
pMgmt->clientRpc = rpcOpen(&rpcInit);
if (pMgmt->clientRpc == NULL) {
dError("failed to init rpc client");
......@@ -256,20 +260,18 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp
static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
if (strcmp(user, INTERNAL_USER) == 0) {
// A simple temporary implementation
char pass[TSDB_PASSWORD_LEN] = {0};
taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
memcpy(secret, pass, TSDB_PASSWORD_LEN);
*spi = 0;
*spi = 1;
*encrypt = 0;
*ckey = 0;
return 0;
} else if (strcmp(user, TSDB_NETTEST_USER) == 0) {
// A simple temporary implementation
char pass[TSDB_PASSWORD_LEN] = {0};
taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
memcpy(secret, pass, TSDB_PASSWORD_LEN);
*spi = 0;
*spi = 1;
*encrypt = 0;
*ckey = 0;
return 0;
......@@ -282,12 +284,12 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char
SDnode *pDnode = parent;
if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) {
// dTrace("get internal auth success");
dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
return 0;
}
if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) {
// dTrace("get auth from internal mnode");
dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
return 0;
}
......@@ -296,13 +298,12 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char
return -1;
}
// dDebug("user:%s, send auth msg to other mnodes", user);
SAuthReq *pReq = rpcMallocCont(sizeof(SAuthReq));
tstrncpy(pReq->user, user, TSDB_USER_LEN);
SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SAuthReq), .msgType = TDMT_MND_AUTH};
SRpcMsg rpcRsp = {0};
dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, pReq->spi, pReq->encrypt);
dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp);
if (rpcRsp.code != 0) {
......@@ -314,7 +315,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char
memcpy(ckey, pRsp->ckey, TSDB_PASSWORD_LEN);
*spi = pRsp->spi;
*encrypt = pRsp->encrypt;
dDebug("user:%s, success to get user auth from other mnodes", user);
dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, pRsp->spi, pRsp->encrypt);
}
rpcFreeCont(rpcRsp.pCont);
......@@ -333,7 +334,7 @@ static int32_t dndInitServer(SDnode *pDnode) {
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = pDnode->cfg.serverPort;
rpcInit.label = "DND-S";
rpcInit.label = "D-S";
rpcInit.numOfThreads = numOfThreads;
rpcInit.cfp = dndProcessRequest;
rpcInit.sessions = pDnode->cfg.maxShellConns;
......
source/dnode/mgmt/impl/test/sut/src/client.cpp
浏览文件 @ 6e358fd4
......@@ -27,8 +27,8 @@ void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; };
tsem_t* TestClient::GetSem() { return &sem; }
bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) {
char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt);
char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt);
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
......@@ -42,7 +42,7 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint
rpcInit.ckey = (char*)"key";
rpcInit.parent = this;
rpcInit.secret = (char*)secretEncrypt;
// rpcInit.spi = 1;
rpcInit.spi = 1;
clientRpc = rpcOpen(&rpcInit);
ASSERT(clientRpc);
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ 6e358fd4
......@@ -328,7 +328,7 @@ typedef struct SMqTopicConsumer {
typedef struct SMqConsumerEp {
int32_t vgId; // -1 for unassigned
SEpSet epset;
SEpSet epSet;
int64_t consumerId; // -1 for unassigned
int64_t lastConsumerHbTs;
int64_t lastVgHbTs;
......@@ -339,7 +339,7 @@ typedef struct SMqConsumerEp {
static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId);
tlen += taosEncodeSEpSet(buf, &pConsumerEp->epset);
tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet);
tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec);
return tlen;
......@@ -347,7 +347,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon
static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId);
buf = taosDecodeSEpSet(buf, &pConsumerEp->epset);
buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet);
buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec);
pConsumerEp->execLen = sizeof(SSubQueryMsg) + pConsumerEp->qExec.contentLen;
......
source/dnode/mnode/impl/src/mndAuth.c
浏览文件 @ 6e358fd4
......@@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "mndAuth.h"
#include "mndUser.h"
static int32_t mndProcessAuthReq(SMnodeMsg *pReq);
......@@ -25,9 +26,34 @@ int32_t mndInitAuth(SMnode *pMnode) {
void mndCleanupAuth(SMnode *pMnode) {}
int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; }
int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
SUserObj *pUser = mndAcquireUser(pMnode, user);
if (pUser == NULL) {
*secret = 0;
mError("user:%s, failed to auth user since %s", user, terrstr());
return -1;
}
static int32_t mndProcessAuthReq(SMnodeMsg *pReq) {
mDebug("user:%s, auth req is processed", pReq->user);
*spi = 1;
*encrypt = 0;
*ckey = 0;
memcpy(secret, pUser->pass, TSDB_PASSWORD_LEN);
mndReleaseUser(pMnode, pUser);
mDebug("user:%s, auth info is returned", user);
return 0;
}
static int32_t mndProcessAuthReq(SMnodeMsg *pReq) {
SAuthReq *pAuth = pReq->rpcMsg.pCont;
int32_t contLen = sizeof(SAuthRsp);
SAuthRsp *pRsp = rpcMallocCont(contLen);
pReq->pCont = pRsp;
pReq->contLen = contLen;
int32_t code = mndRetriveAuth(pReq->pMnode, pAuth->user, &pRsp->spi, &pRsp->encrypt, pRsp->secret, pRsp->ckey);
mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->user, pAuth->spi, pAuth->encrypt, pAuth->user);
return code;
}
\ No newline at end of file
source/dnode/mnode/impl/src/mndSubscribe.c
浏览文件 @ 6e358fd4
......@@ -117,7 +117,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
// persist msg
STransAction action = {0};
action.epSet = pCEp->epset;
action.epSet = pCEp->epSet;
action.pCont = reqStr;
action.contLen = tlen;
action.msgType = TDMT_VND_MQ_SET_CONN;
......@@ -142,36 +142,25 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
}
static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) {
SMqConsumerEp CEp;
CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1;
//convert phyplan to dag
SQueryDag *pDag = qStringToDag(pTopic->physicalPlan);
SArray *pArray;
if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) {
return -1;
}
int32_t sz = taosArrayGetSize(pArray);
//convert dag to msg
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp CEp;
CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1;
STaskInfo* pTaskInfo = taosArrayGet(pArray, i);
int32_t vgId = pTaskInfo->addr.nodeId;
SEpSet epSet;
tConvertQueryAddrToEpSet(&epSet, &pTaskInfo->addr);
}
/*pTopic->physicalPlan;*/
SVgObj *pVgroup = NULL;
SSdb *pSdb = pMnode->pSdb;
void *pIter = sdbFetch(pSdb, SDB_VGROUP, NULL, (void **)&pVgroup);
while (pIter != NULL) {
if (pVgroup->dbUid == pTopic->dbUid) {
CEp.epset = mndGetVgroupEpset(pMnode, pVgroup);
CEp.vgId = pVgroup->vgId;
tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr);
CEp.vgId = pTaskInfo->addr.nodeId;
taosArrayPush(unassignedVg, &CEp);
}
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
}
return 0;
qDestroyQueryDag(pDag);
return 0;
}
static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer,
......
source/dnode/mnode/impl/src/mndUser.c
浏览文件 @ 6e358fd4
......@@ -60,9 +60,9 @@ void mndCleanupUser(SMnode *pMnode) {}
static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) {
SUserObj userObj = {0};
taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass);
tstrncpy(userObj.user, user, TSDB_USER_LEN);
tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass);
userObj.createdTime = taosGetTimestampMs();
userObj.updateTime = userObj.createdTime;
......@@ -202,7 +202,7 @@ SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) {
SSdb *pSdb = pMnode->pSdb;
SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName);
if (pUser == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
terrno = TSDB_CODE_MND_USER_NOT_EXIST;
}
return pUser;
}
......@@ -214,9 +214,9 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) {
static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pReq) {
SUserObj userObj = {0};
taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass);
tstrncpy(userObj.user, user, TSDB_USER_LEN);
tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass);
userObj.createdTime = taosGetTimestampMs();
userObj.updateTime = userObj.createdTime;
userObj.superUser = 0;
......@@ -351,8 +351,9 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) {
SUserObj newUser = {0};
memcpy(&newUser, pUser, sizeof(SUserObj));
memset(pUser->pass, 0, sizeof(pUser->pass));
taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass);
char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)pAlter->pass, strlen(pAlter->pass), pass);
memcpy(pUser->pass, pass, TSDB_PASSWORD_LEN);
newUser.updateTime = taosGetTimestampMs();
int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pReq);
......
source/dnode/mnode/sdb/src/sdbHash.c
浏览文件 @ 6e358fd4
......@@ -152,6 +152,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
if (insertFp != NULL) {
code = (*insertFp)(pSdb, pRow->pObj);
if (code != 0) {
code = terrno;
taosWLockLatch(pLock);
taosHashRemove(hash, pRow->pObj, keySize);
taosWUnLockLatch(pLock);
......
source/dnode/vnode/CMakeLists.txt
浏览文件 @ 6e358fd4
......@@ -26,6 +26,7 @@ target_link_libraries(
PUBLIC tfs
PUBLIC wal
PUBLIC scheduler
PUBLIC executor
PUBLIC qworker
)
......
source/dnode/vnode/inc/tq.h
浏览文件 @ 6e358fd4
......@@ -17,6 +17,8 @@
#define _TD_TQ_H_
#include "common.h"
#include "executor.h"
#include "vnode.h"
#include "mallocator.h"
#include "meta.h"
#include "os.h"
......@@ -81,27 +83,12 @@ typedef struct STqSubscribeReq {
int64_t topic[];
} STqSubscribeReq;
typedef struct STqSubscribeRsp {
STqMsgHead head;
int64_t vgId;
char ep[TSDB_EP_LEN]; // TSDB_EP_LEN
} STqSubscribeRsp;
typedef struct STqHeartbeatReq {
} STqHeartbeatReq;
typedef struct STqHeartbeatRsp {
} STqHeartbeatRsp;
typedef struct STqTopicVhandle {
int64_t topicId;
// executor for filter
void* filterExec;
// callback for mnode
// trigger when vnode list associated topic change
void* (*mCallback)(void*, void*);
} STqTopicVhandle;
#define TQ_BUFFER_SIZE 8
typedef struct STqExec {
......@@ -165,7 +152,7 @@ typedef struct STqTaskItem {
int8_t status;
int64_t offset;
void* dst;
SSubQueryMsg* pMsg;
qTaskInfo_t task;
} STqTaskItem;
// new version
......@@ -198,10 +185,6 @@ typedef struct STqQueryMsg {
struct STqQueryMsg* next;
} STqQueryMsg;
typedef struct STqCfg {
// TODO
} STqCfg;
typedef struct STqMemRef {
SMemAllocatorFactory* pAllocatorFactory;
SMemAllocator* pAllocator;
......@@ -298,6 +281,7 @@ typedef struct STQ {
STqMemRef tqMemRef;
STqMetaStore* tqMeta;
SWal* pWal;
SMeta* pMeta;
} STQ;
typedef struct STqMgmt {
......@@ -312,13 +296,15 @@ int tqInit();
void tqCleanUp();
// open in each vnode
STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac);
STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac);
void tqClose(STQ*);
// void* will be replace by a msg type
int tqPushMsg(STQ*, void* msg, int64_t version);
int tqCommit(STQ*);
int tqSetCursor(STQ*, STqSetCurReq* pMsg);
#if 0
int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp);
int tqSetCursor(STQ*, STqSetCurReq* pMsg);
......@@ -334,23 +320,6 @@ int tqSendLaunchQuery(STqMsgItem*, int64_t offset);
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp);
int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq);
typedef struct STqReadHandle {
int64_t ver;
SSubmitMsg* pMsg;
SSubmitBlk* pBlock;
SSubmitMsgIter msgIter;
SSubmitBlkIter blkIter;
SMeta* pMeta;
SArray* pColumnIdList;
} STqReadHandle;
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList);
void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle* pHandle);
int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo);
// return SArray<SColumnInfoData>
SArray* tqRetrieveDataBlock(STqReadHandle* pHandle);
#ifdef __cplusplus
}
#endif
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ 6e358fd4
......@@ -22,7 +22,6 @@
#include "meta.h"
#include "tarray.h"
#include "tfs.h"
#include "tq.h"
#include "tsdb.h"
#include "wal.h"
......@@ -35,6 +34,12 @@ typedef struct SVnode SVnode;
typedef struct SDnode SDnode;
typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq);
typedef struct STqCfg {
// TODO
int32_t reserved;
} STqCfg;
typedef struct SVnodeCfg {
int32_t vgId;
SDnode *pDnode;
......@@ -61,6 +66,17 @@ typedef struct {
PutReqToVQueryQFp putReqToVQueryQFp;
} SVnodeOpt;
typedef struct STqReadHandle {
int64_t ver;
uint64_t tbUid;
SSubmitMsg* pMsg;
SSubmitBlk* pBlock;
SSubmitMsgIter msgIter;
SSubmitBlkIter blkIter;
SMeta* pMeta;
SArray* pColIdList;
} STqReadHandle;
/* ------------------------ SVnode ------------------------ */
/**
* @brief Initialize the vnode module
......@@ -180,6 +196,25 @@ int32_t vnodeCompact(SVnode *pVnode);
int32_t vnodeSync(SVnode *pVnode);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
/* ------------------------- TQ QUERY -------------------------- */
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta);
static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) {
pReadHandle->pColIdList = pColIdList;
}
static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, uint64_t tbUid) {
pHandle->tbUid = tbUid;
}
void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle* pHandle);
int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo);
// return SArray<SColumnInfoData>
SArray* tqRetrieveDataBlock(STqReadHandle* pHandle);
#ifdef __cplusplus
}
#endif
......
source/dnode/vnode/src/inc/tqInt.h
浏览文件 @ 6e358fd4
......@@ -17,6 +17,7 @@
#define _TD_TQ_INT_H_
#include "tq.h"
#include "meta.h"
#include "tlog.h"
#include "trpc.h"
#ifdef __cplusplus
......
source/dnode/vnode/src/inc/vnd.h
浏览文件 @ 6e358fd4
......@@ -24,6 +24,7 @@
#include "tlockfree.h"
#include "tmacro.h"
#include "wal.h"
#include "tq.h"
#include "vnode.h"
......
source/dnode/vnode/src/meta/metaTbCfg.c
浏览文件 @ 6e358fd4
......@@ -32,7 +32,7 @@ size_t metaEncodeTbObjFromTbOptions(const STbCfg *pTbOptions, void *pBuf, size_t
switch (pTbOptions->type) {
case META_SUPER_TABLE:
tlen += taosEncodeFixedU64(ppBuf, pTbOptions->stbCfg.suid);
tlen += tdEncodeSchema(ppBuf, pTbOptions->stbCfg.pTagSchema);
tlen += tdEncodeSchema(ppBuf, (STSchema *)pTbOptions->stbCfg.pTagSchema);
// TODO: encode schema version array
break;
case META_CHILD_TABLE:
......
source/dnode/vnode/src/tq/tq.c
浏览文件 @ 6e358fd4
......@@ -13,9 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "../../../../../include/libs/executor/executor.h"
#include "tqInt.h"
#include "tqMetaStore.h"
#include "tcompare.h"
// static
// read next version data
......@@ -51,7 +51,7 @@ void tqCleanUp() {
taosTmrCleanUp(tqMgmt.timer);
}
STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
STQ* pTq = malloc(sizeof(STQ));
if (pTq == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
......@@ -59,6 +59,8 @@ STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory
}
pTq->path = strdup(path);
pTq->tqConfig = tqConfig;
pTq->pWal = pWal;
pTq->pMeta = pMeta;
#if 0
pTq->tqMemRef.pAllocatorFactory = allocFac;
pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
......@@ -608,54 +610,92 @@ int tqItemSSize() {
}
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
SMqCVConsumeReq* pReq = pMsg->pCont;
SMqConsumeReq* pReq = pMsg->pCont;
SRpcMsg rpcMsg;
int64_t reqId = pReq->reqId;
int64_t consumerId = pReq->consumerId;
int64_t offset = pReq->offset;
int64_t reqOffset = pReq->offset;
int64_t fetchOffset = reqOffset;
int64_t blockingTime = pReq->blockingTime;
int rspLen = 0;
STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
int sz = taosArrayGetSize(pConsumer->topics);
for (int i = 0 ; i < sz; i++) {
STqTopicHandle *pHandle = taosArrayGet(pConsumer->topics, i);
for (int i = 0; i < sz; i++) {
STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i);
int8_t pos = offset % TQ_BUFFER_SIZE;
int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1);
if (old == 1) {
int8_t pos;
int8_t skip = 0;
SWalHead* pHead;
while (1) {
pos = fetchOffset % TQ_BUFFER_SIZE;
skip = atomic_val_compare_exchange_8(&pTopic->buffer.output[pos].status, 0, 1);
if (skip == 1) {
// do nothing
continue;
break;
}
if (walReadWithHandle(pHandle->pReadhandle, offset) < 0) {
// TODO
if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
// check err
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
skip = 1;
break;
}
SWalHead* pHead = pHandle->pReadhandle->pHead;
while (pHead->head.msgType != TDMT_VND_SUBMIT) {
// read until find TDMT_VND_SUBMIT
pHead = pTopic->pReadhandle->pHead;
if (pHead->head.msgType == TDMT_VND_SUBMIT) {
break;
}
if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
skip = 1;
break;
}
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
fetchOffset++;
}
if (skip == 1) continue;
SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body;
qTaskInfo_t task = pTopic->buffer.output[pos].task;
SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg;
qSetStreamInput(task, pCont);
// TODO: launch query and get output data
void* outputData;
pHandle->buffer.output[pos].dst = outputData;
if (pHandle->buffer.firstOffset == -1
|| pReq->offset < pHandle->buffer.firstOffset) {
pHandle->buffer.firstOffset = pReq->offset;
//SArray<SSDataBlock>
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
while (1) {
SSDataBlock* pDataBlock;
uint64_t ts;
if (qExecTask(task, &pDataBlock, &ts) < 0) {
break;
}
if (pHandle->buffer.lastOffset == -1
|| pReq->offset > pHandle->buffer.lastOffset) {
pHandle->buffer.lastOffset = pReq->offset;
if (pDataBlock != NULL) {
taosArrayPush(pRes, pDataBlock);
} else {
break;
}
}
atomic_store_8(&pTopic->buffer.output[pos].status, 0);
if (taosArrayGetSize(pRes) == 0) {
taosArrayDestroy(pRes);
fetchOffset++;
continue;
}
atomic_store_8(&pHandle->buffer.output[pos].status, 1);
pTopic->buffer.output[pos].dst = pRes;
if (pTopic->buffer.firstOffset == -1 || pReq->offset < pTopic->buffer.firstOffset) {
pTopic->buffer.firstOffset = pReq->offset;
}
if (pTopic->buffer.lastOffset == -1 || pReq->offset > pTopic->buffer.lastOffset) {
pTopic->buffer.lastOffset = pReq->offset;
}
// put output into rsp
}
// launch query
// get result
SMqCvConsumeRsp* pRsp;
return 0;
}
......@@ -675,29 +715,22 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) {
strcpy(pTopic->sql, pReq->sql);
strcpy(pTopic->logicalPlan, pReq->logicalPlan);
strcpy(pTopic->physicalPlan, pReq->physicalPlan);
SArray *pArray;
//TODO: deserialize to SQueryDag
SQueryDag *pDag;
// convert to task
if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) {
// TODO: handle error
}
STaskInfo *pInfo = taosArrayGet(pArray, 0);
SArray* pTasks;
schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE);
pTopic->buffer.firstOffset = -1;
pTopic->buffer.lastOffset = -1;
pTopic->pReadhandle = walOpenReadHandle(pTq->pWal);
if (pTopic->pReadhandle == NULL) {
}
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
SSubQueryMsg* pMsg = taosArrayGet(pTasks, i);
pTopic->buffer.output[i].pMsg = pMsg;
pTopic->buffer.output[i].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta);
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&pReq->msg, pReadHandle);
}
pTopic->pReadhandle = walOpenReadHandle(pTq->pWal);
// write mq meta
return 0;
}
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList) {
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle));
if (pReadHandle == NULL) {
return NULL;
......@@ -705,7 +738,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList) {
pReadHandle->pMeta = pMeta;
pReadHandle->pMsg = NULL;
pReadHandle->ver = -1;
pReadHandle->pColumnIdList = pColumnIdList;
pReadHandle->pColIdList = NULL;
return NULL;
}
......@@ -717,20 +750,18 @@ void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ve
}
bool tqNextDataBlock(STqReadHandle* pHandle) {
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
return false;
while (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) >= 0) {
if (pHandle->tbUid == pHandle->pBlock->uid) return true;
}
return true;
return false;
}
int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) {
SMemRow row;
int32_t sversion = pHandle->pBlock->sversion;
SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);
pBlockInfo->numOfCols = pSchema->nCols;
/*int32_t sversion = pHandle->pBlock->sversion;*/
/*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/
pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList);
pBlockInfo->rows = pHandle->pBlock->numOfRows;
pBlockInfo->uid = pHandle->pBlock->uid;
// TODO: filter out unused column
return 0;
}
......@@ -763,32 +794,3 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
taosArrayPush(pArray, &colInfo);
return pArray;
}
static qTaskInfo_t createExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle) {
if (pMsg == NULL || pStreamBlockReadHandle == NULL) {
return NULL;
}
// print those info into log
pMsg->sId = be64toh(pMsg->sId);
pMsg->queryId = be64toh(pMsg->queryId);
pMsg->taskId = be64toh(pMsg->taskId);
pMsg->contentLen = ntohl(pMsg->contentLen);
struct SSubplan *plan = NULL;
int32_t code = qStringToSubplan(pMsg->msg, &plan);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
}
qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(pStreamBlockReadHandle, 0, plan, &pTaskInfo, NULL);
if (code != TSDB_CODE_SUCCESS) {
// TODO: destroy SSubplan & pTaskInfo
terrno = code;
return NULL;
}
return pTaskInfo;
}
\ No newline at end of file
source/dnode/vnode/src/vnd/vnodeMain.c
浏览文件 @ 6e358fd4
......@@ -127,7 +127,7 @@ static int vnodeOpenImpl(SVnode *pVnode) {
// Open TQ
sprintf(dir, "%s/tq", pVnode->path);
pVnode->pTq = tqOpen(dir, pVnode->pWal, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode));
pVnode->pTq = tqOpen(dir, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode));
if (pVnode->pTq == NULL) {
// TODO: handle error
return -1;
......
source/dnode/vnode/src/vnd/vnodeQuery.c
浏览文件 @ 6e358fd4
......@@ -19,7 +19,10 @@
static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg);
static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, &pVnode->pQuery, pVnode, vnodePutReqToVQueryQ); }
int vnodeQueryOpen(SVnode *pVnode) {
return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode,
(putReqToQueryQFp)vnodePutReqToVQueryQ);
}
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vTrace("query message is processing");
......@@ -153,6 +156,11 @@ _exit:
return 0;
}
static void freeItemHelper(void* pItem) {
char* p = *(char**)pItem;
free(p);
}
/**
* @param pVnode
* @param pMsg
......@@ -166,17 +174,20 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t totalLen = 0;
int32_t numOfTables = 0;
while ((name = metaTbCursorNext(pCur)) != NULL) {
if (numOfTables < 1000) { // TODO: temp get tables of vnode, and should del when show tables commad ok.
if (numOfTables < 10000) { // TODO: temp get tables of vnode, and should del when show tables commad ok.
taosArrayPush(pArray, &name);
totalLen += strlen(name);
} else {
tfree(name);
}
numOfTables++;
}
// TODO: temp debug, and should del when show tables command ok
vError("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables);
if (numOfTables > 1000) {
numOfTables = 1000;
vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables);
if (numOfTables > 10000) {
numOfTables = 10000;
}
metaCloseTbCursor(pCur);
......@@ -211,6 +222,6 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
};
rpcSendResponse(&rpcMsg);
taosArrayDestroy(pArray);
taosArrayDestroyEx(pArray, freeItemHelper);
return 0;
}
source/dnode/vnode/src/vnd/vnodeWrite.c
浏览文件 @ 6e358fd4
......@@ -83,8 +83,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i);
if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) {
// TODO: handle error
vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name);
}
vTrace("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name);
free(pCreateTbReq->name);
if (pCreateTbReq->type == TD_SUPER_TABLE) {
free(pCreateTbReq->stbCfg.pSchema);
......@@ -95,6 +95,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
free(pCreateTbReq->ntbCfg.pSchema);
}
}
vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray));
taosArrayDestroy(vCreateTbBatchReq.pArray);
break;
......@@ -129,6 +131,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
// TODO: handle error
}
}
return 0;
}
......
source/libs/executor/inc/executorimpl.h
浏览文件 @ 6e358fd4
......@@ -372,11 +372,14 @@ typedef struct STaskParam {
typedef struct SExchangeInfo {
SArray *pSources;
uint64_t bytes; // total load bytes from remote
tsem_t ready;
void *pTransporter;
SRetrieveTableRsp *pRsp;
SSDataBlock *pResult;
int32_t current;
uint64_t rowsOfCurrentSource;
uint64_t bytes; // total load bytes from remote
uint64_t totalRows;
} SExchangeInfo;
typedef struct STableScanInfo {
......
source/libs/executor/src/executor.c
浏览文件 @ 6e358fd4
......@@ -12,3 +12,81 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executor.h"
#include "tq.h"
#include "executorimpl.h"
#include "planner.h"
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, uint64_t reqId) {
ASSERT(pOperator != NULL);
if (pOperator->operatorType != OP_StreamScan) {
if (pOperator->numOfDownstream == 0) {
qError("failed to find stream scan operator to set the input data block, reqId:0x%" PRIx64, reqId);
return TSDB_CODE_QRY_APP_ERROR;
}
if (pOperator->numOfDownstream > 1) { // not handle this in join query
qError("join not supported for stream block scan, reqId:0x%" PRIx64, reqId);
return TSDB_CODE_QRY_APP_ERROR;
}
return doSetStreamBlock(pOperator->pDownstream[0], input, reqId);
} else {
SStreamBlockScanInfo* pInfo = pOperator->info;
tqReadHandleSetMsg(pInfo->readerHandle, input, 0);
return TSDB_CODE_SUCCESS;
}
}
int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) {
if (tinfo == NULL) {
return TSDB_CODE_QRY_APP_ERROR;
}
if (input == NULL) {
return TSDB_CODE_SUCCESS;
}
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo;
int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
qError("failed to set the stream block data, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo));
} else {
qDebug("set the stream block successfully, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo));
}
return code;
}
qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle) {
if (pMsg == NULL || streamReadHandle == NULL) {
return NULL;
}
// print those info into log
#if 0
pMsg->sId = pMsg->sId;
pMsg->queryId = pMsg->queryId;
pMsg->taskId = pMsg->taskId;
pMsg->contentLen = pMsg->contentLen;
#endif
struct SSubplan* plan = NULL;
int32_t code = qStringToSubplan(pMsg->msg, &plan);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
}
qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(streamReadHandle, 0, plan, &pTaskInfo, NULL);
if (code != TSDB_CODE_SUCCESS) {
// TODO: destroy SSubplan & pTaskInfo
terrno = code;
return NULL;
}
return pTaskInfo;
}
source/libs/executor/src/executorMain.c
浏览文件 @ 6e358fd4
......@@ -164,13 +164,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
return TSDB_CODE_SUCCESS;
}
// STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv;
// if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) {
// qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo));
// setTaskStatus(pTaskInfo, TASK_COMPLETED);
// return doBuildResCheck(pTaskInfo);
// }
// error occurs, record the error code and return to client
int32_t ret = setjmp(pTaskInfo->env);
if (ret != TSDB_CODE_SUCCESS) {
......
source/libs/executor/src/executorimpl.c
浏览文件 @ 6e358fd4
......@@ -27,6 +27,7 @@
#include "thash.h"
#include "ttypes.h"
#include "query.h"
#include "vnode.h"
#include "tsdb.h"
#define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN)
......@@ -4957,6 +4958,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
STableScanInfo *pTableScanInfo = pOperator->info;
SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo;
if (pTableScanInfo->pTsdbReadHandle == NULL) {
return NULL;
}
SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo;
*newgroup = false;
......@@ -5143,32 +5148,41 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo;
*newgroup = false;
if (pExchangeInfo->pRsp != NULL && pExchangeInfo->pRsp->completed == 1) {
size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
if (pExchangeInfo->current >= totalSources) {
return NULL;
}
SResFetchReq *pMsg = calloc(1, sizeof(SResFetchReq));
SResFetchReq* pMsg = NULL;
SMsgSendInfo* pMsgSendInfo = NULL;
while(1) {
pMsg = calloc(1, sizeof(SResFetchReq));
if (NULL == pMsg) { // todo handle malloc error
pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _error;
}
SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, 0);
SEpSet epSet = {0};
SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
SEpSet epSet = {0};
epSet.numOfEps = pSource->addr.numOfEps;
epSet.port[0] = pSource->addr.epAddr[0].port;
tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0]));
qDebug("QID:0x%" PRIx64 " build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, epSet.fqdn[0], pSource->taskId, pExchangeInfo->current, totalSources);
pMsg->header.vgId = htonl(pSource->addr.nodeId);
pMsg->sId = htobe64(pSource->schedId);
pMsg->taskId = htobe64(pSource->taskId);
pMsg->queryId = htobe64(pTaskInfo->id.queryId);
// send the fetch remote task result reques
SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo));
pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
qError("QID:%"PRIx64" calloc %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
qError("QID:0x%" PRIx64 " prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _error;
}
......@@ -5183,21 +5197,33 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo);
tsem_wait(&pExchangeInfo->ready);
if (pExchangeInfo->pRsp->numOfRows == 0) {
SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp;
if (pRsp->numOfRows == 0) {
qDebug("QID:0x%"PRIx64" vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1,
pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows);
pExchangeInfo->rowsOfCurrentSource = 0;
pExchangeInfo->current += 1;
if (pExchangeInfo->current >= totalSources) {
return NULL;
} else {
continue;
}
}
SSDataBlock* pRes = pExchangeInfo->pResult;
char* pData = pExchangeInfo->pRsp->data;
char* pData = pRsp->data;
for(int32_t i = 0; i < pOperator->numOfOutput; ++i) {
for (int32_t i = 0; i < pOperator->numOfOutput; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i);
char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pExchangeInfo->pRsp->numOfRows);
char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pRsp->numOfRows);
if (tmp == NULL) {
goto _error;
}
size_t len = pExchangeInfo->pRsp->numOfRows * pColInfoData->info.bytes;
size_t len = pRsp->numOfRows * pColInfoData->info.bytes;
memcpy(tmp, pData, len);
pColInfoData->pData = tmp;
......@@ -5205,9 +5231,27 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
}
pRes->info.numOfCols = pOperator->numOfOutput;
pRes->info.rows = pExchangeInfo->pRsp->numOfRows;
pRes->info.rows = pRsp->numOfRows;
pExchangeInfo->totalRows += pRsp->numOfRows;
pExchangeInfo->bytes += pRsp->compLen;
pExchangeInfo->rowsOfCurrentSource += pRsp->numOfRows;
if (pRsp->completed == 1) {
qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows, pExchangeInfo->bytes,
pExchangeInfo->current + 1, totalSources);
pExchangeInfo->rowsOfCurrentSource = 0;
pExchangeInfo->current += 1;
} else {
qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->bytes);
}
return pExchangeInfo->pResult;
}
_error:
tfree(pMsg);
......@@ -5258,7 +5302,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
rpcInit.user = (char *)"root";
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.ckey = "key";
// rpcInit.spi = 1;
rpcInit.spi = 1;
rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6";
pInfo->pTransporter = rpcOpen(&rpcInit);
......@@ -5407,7 +5451,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt
return pOperator;
}
SOperatorInfo* createStreamBlockScanOperatorInfo(void *pStreamBlockHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, uint64_t uid, SExecTaskInfo* pTaskInfo) {
SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo));
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
......@@ -5417,16 +5461,29 @@ SOperatorInfo* createStreamBlockScanOperatorInfo(void *pStreamBlockHandle, int32
return NULL;
}
pInfo->readerHandle = pStreamBlockHandle;
int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprInfo);
SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t));
for(int32_t i = 0; i < numOfOutput; ++i) {
SExprInfo* pExpr = taosArrayGetP(pExprInfo, i);
taosArrayPush(pColList, &pExpr->pExpr->pSchema[0].colId);
}
// set the extract column id to streamHandle
tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList);
tqReadHandleSetTbUid(streamReadHandle, uid);
pInfo->readerHandle = streamReadHandle;
pOperator->name = "StreamBlockScanOperator";
pOperator->operatorType = OP_StreamBlockScan;
pOperator->operatorType = OP_StreamScan;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->exec = doStreamBlockScan;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
}
......@@ -7704,6 +7761,9 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask
} else if (pPhyNode->info.type == OP_Exchange) {
SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode;
return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo);
} else if (pPhyNode->info.type == OP_StreamScan) {
SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table.
return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pScanPhyNode->uid, pTaskInfo);
}
}
......@@ -7768,11 +7828,12 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode,
if (groupInfo.numOfTables == 0) {
code = 0;
// qDebug("no table qualified for query, reqId:0x%"PRIx64, (*pTask)->id.queryId);
qDebug("no table qualified for query, reqId:0x%"PRIx64, queryId);
goto _error;
}
return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId);
_error:
terrno = code;
return NULL;
......
source/libs/executor/test/executorTests.cpp
浏览文件 @ 6e358fd4
......@@ -17,8 +17,9 @@
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
......@@ -220,3 +221,5 @@ TEST(testCase, build_executor_tree_Test) {
DataSinkHandle sinkHandle = nullptr;
int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo, &sinkHandle);
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/parser/src/dCDAstProcess.c
浏览文件 @ 6e358fd4
......@@ -26,7 +26,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out
const char* msg4 = "pattern is invalid";
const char* msg5 = "database name is empty";
const char* msg6 = "pattern string is empty";
const char* msg7 = "db is not specified";
const char* msg7 = "database not specified";
/*
* database prefix in pInfo->pMiscInfo->a[0]
* wildcard in like clause in pInfo->pMiscInfo->a[1]
......@@ -50,7 +50,11 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out
char dbFname[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbFname);
catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array);
int32_t code = catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return code;
}
SVgroupInfo* info = taosArrayGet(array, 0);
pShowReq->head.vgId = htonl(info->vgId);
......@@ -64,7 +68,6 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out
*outputLen = sizeof(SVShowTablesReq);
*output = pShowReq;
*pExtension = array;
} else {
if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP) {
......
source/libs/parser/src/parser.c
浏览文件 @ 6e358fd4
......@@ -36,25 +36,29 @@ bool qIsDdlQuery(const SQueryNode* pQueryNode) {
}
int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
int32_t code = TSDB_CODE_SUCCESS;
SSqlInfo info = doGenerateAST(pCxt->pSql);
if (!info.valid) {
strncpy(pCxt->pMsg, info.msg, pCxt->msgLen);
terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
return terrno;
code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
goto _end;
}
if (!isDqlSqlStatement(&info)) {
if (info.type == TSDB_SQL_CREATE_TABLE) {
SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen);
if (pModifStmtInfo == NULL) {
return terrno;
code = terrno;
goto _end;
}
*pQuery = (SQueryNode*)pModifStmtInfo;
} else {
SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen);
if (pDcl == NULL) {
return terrno;
code = terrno;
goto _end;
}
*pQuery = (SQueryNode*)pDcl;
......@@ -63,21 +67,22 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
} else {
SQueryStmtInfo* pQueryInfo = createQueryInfo();
if (pQueryInfo == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // set correct error code.
return terrno;
code = TSDB_CODE_QRY_OUT_OF_MEMORY; // set correct error code.
goto _end;
}
int32_t code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen);
code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen);
if (code == TSDB_CODE_SUCCESS) {
*pQuery = (SQueryNode*)pQueryInfo;
} else {
terrno = code;
return code;
goto _end;
}
}
_end:
destroySqlInfo(&info);
return TSDB_CODE_SUCCESS;
terrno = code;
return code;
}
int32_t qParseQuerySql(SParseContext* pCxt, SQueryNode** pQueryNode) {
......@@ -247,5 +252,6 @@ void qDestroyQuery(SQueryNode* pQueryNode) {
SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode;
taosArrayDestroy(pModifInfo->pDataBlocks);
}
tfree(pQueryNode);
}
source/libs/parser/test/mockCatalog.cpp
浏览文件 @ 6e358fd4
......@@ -18,8 +18,14 @@
#include <iostream>
#include "stub.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat"
#include "addr_any.h"
#pragma GCC diagnostic pop
namespace {
void generateTestT1(MockCatalogService* mcs) {
......
source/libs/parser/test/parserTests.cpp
浏览文件 @ 6e358fd4
......@@ -17,8 +17,9 @@
#include <gtest/gtest.h>
#include <iostream>
#include "tglobal.h"
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
......@@ -782,3 +783,5 @@ TEST(testCase, create_user_Test) {
destroySqlInfo(&info1);
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/parser/test/plannerTest.cpp
浏览文件 @ 6e358fd4
......@@ -17,8 +17,9 @@
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
......@@ -204,3 +205,5 @@ TEST(testCase, displayPlan) {
// TableScan(t.1abc #110) time_range: -9223372036854775808 - 9223372036854775807
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/parser/test/tokenizerTest.cpp
浏览文件 @ 6e358fd4
#include <gtest/gtest.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
......@@ -722,3 +723,4 @@ TEST(testCase, extractMeta_test) {
destroySqlInfo(&info1);
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/planner/inc/plannerInt.h
浏览文件 @ 6e358fd4
......@@ -106,7 +106,7 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
*/
int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId);
int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId);
void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource);
int32_t subPlanToString(const SSubplan *pPhyNode, char** str, int32_t* len);
int32_t stringToSubplan(const char* str, SSubplan** subplan);
......
source/libs/planner/src/physicalPlan.c
浏览文件 @ 6e358fd4
......@@ -261,14 +261,14 @@ static void vgroupMsgToEpSet(const SVgroupMsg* vg, SQueryNodeAddr* execNode) {
return;
}
static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
SVgroupsInfo* pVgroupList = pTable->pMeta->vgroupList;
static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) {
SVgroupsInfo* pVgroupList = pTableInfo->pMeta->vgroupList;
for (int32_t i = 0; i < pVgroupList->numOfVgroups; ++i) {
STORE_CURRENT_SUBPLAN(pCxt);
SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN);
subplan->msgType = TDMT_VND_QUERY;
vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execNode);
subplan->pNode = createMultiTableScanNode(pPlanNode, pTable);
vgroupMsgToEpSet(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode);
subplan->pNode = createMultiTableScanNode(pPlanNode, pTableInfo);
subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode, subplan->pNode);
RECOVERY_CURRENT_SUBPLAN(pCxt);
}
......@@ -389,13 +389,14 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) {
// todo deal subquery
}
int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId) {
int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId) {
TRY(TSDB_MAX_TAG_CONDITIONS) {
SPlanContext context = {
.pCatalog = pCatalog,
.pDag = validPointer(calloc(1, sizeof(SQueryDag))),
.pCurrentSubplan = NULL,
.nextId = {.queryId = requestId},
//The unsigned Id starting from 1 would be better
.nextId = {.queryId = requestId, .subplanId = 1, .templateId = 1},
};
*pDag = context.pDag;
......@@ -408,6 +409,17 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD
terrno = code;
return TSDB_CODE_FAILED;
} END_TRY
// traverse the dag again to acquire the execution node.
if (pNodeList != NULL) {
SArray** pSubLevel = taosArrayGetLast((*pDag)->pSubplans);
size_t num = taosArrayGetSize(*pSubLevel);
for (int32_t j = 0; j < num; ++j) {
SSubplan* pPlan = taosArrayGetP(*pSubLevel, j);
taosArrayPush(pNodeList, &pPlan->execNode);
}
}
return TSDB_CODE_SUCCESS;
}
......
source/libs/planner/src/planner.c
浏览文件 @ 6e358fd4
......@@ -16,6 +16,8 @@
#include "parser.h"
#include "plannerInt.h"
static void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols);
static void destroyDataSinkNode(SDataSink* pSinkNode) {
if (pSinkNode == NULL) {
return;
......@@ -56,7 +58,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) {
tfree(pDag);
}
int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) {
int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId) {
SQueryPlanNode* pLogicPlan;
int32_t code = createQueryPlan(pNode, &pLogicPlan);
if (TSDB_CODE_SUCCESS != code) {
......@@ -76,17 +78,31 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag,
return code;
}
code = createDag(pLogicPlan, NULL, pDag, requestId);
code = createDag(pLogicPlan, NULL, pDag, pNodeList, requestId);
if (TSDB_CODE_SUCCESS != code) {
destroyQueryPlan(pLogicPlan);
qDestroyQueryDag(*pDag);
return code;
}
extractResSchema(pDag, pResSchema, numOfCols);
destroyQueryPlan(pLogicPlan);
return TSDB_CODE_SUCCESS;
}
void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema,
int32_t* numOfCols) { // extract the final result schema
SArray* pTopSubplan = taosArrayGetP((*pDag)->pSubplans, 0);
SSubplan* pPlan = taosArrayGetP(pTopSubplan, 0);
SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema);
*numOfCols = pDataBlockSchema->numOfCols;
*pResSchema = calloc(pDataBlockSchema->numOfCols, sizeof(SSchema));
memcpy((*pResSchema), pDataBlockSchema->pSchema, pDataBlockSchema->numOfCols * sizeof(SSchema));
}
void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) {
setSubplanExecutionNode(subplan, templateId, pSource);
}
......
source/libs/planner/test/phyPlanTests.cpp
浏览文件 @ 6e358fd4
......@@ -61,7 +61,7 @@ protected:
int32_t run() {
SQueryDag* dag = nullptr;
uint64_t requestId = 20;
int32_t code = createDag(logicPlan_.get(), nullptr, &dag, requestId);
int32_t code = createDag(logicPlan_.get(), nullptr, &dag, NULL, requestId);
dag_.reset(dag);
return code;
}
......@@ -78,9 +78,9 @@ protected:
SQueryDag* dag = nullptr;
uint64_t requestId = 20;
SSchema *schema = NULL;
uint32_t numOfOutput = 0;
int32_t numOfOutput = 0;
code = qCreateQueryDag(query, &dag, requestId);
code = qCreateQueryDag(query, &dag, &schema, &numOfOutput, nullptr, requestId);
dag_.reset(dag);
return code;
}
......
source/libs/planner/test/plannerTests.cpp
浏览文件 @ 6e358fd4
......@@ -22,8 +22,8 @@
#include "parser.h"
#include "mockCatalog.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
......@@ -103,3 +103,5 @@ TEST(testCase, planner_test) {
// qParserCleanupMetaRequestInfo(&req);
// destroySqlInfo(&info1);
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/qcom/src/querymsg.c
浏览文件 @ 6e358fd4
......@@ -18,6 +18,9 @@
#include "query.h"
#include "trpc.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat-truncation"
int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0};
int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize) = {0};
......@@ -288,7 +291,4 @@ void initQueryModuleMsgHandle() {
queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp;
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/qcom/test/queryTest.cpp
浏览文件 @ 6e358fd4
......@@ -18,8 +18,9 @@
#include "tmsg.h"
#include "query.h"
#include "trpc.h"
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
......@@ -82,3 +83,5 @@ TEST(testCase, error_in_async_test) {
usleep(1000);
printf("Error code:%d after asynchronously exec function\n", code);
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/qworker/inc/qworkerInt.h
浏览文件 @ 6e358fd4
......@@ -171,17 +171,17 @@ typedef struct SQWorkerMgmt {
#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_ELOG_E(param) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0)
......@@ -219,8 +219,7 @@ typedef struct SQWorkerMgmt {
} \
} while (0)
int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code);
#ifdef __cplusplus
}
......
source/libs/qworker/test/qworkerTests.cpp
浏览文件 @ 6e358fd4
......@@ -16,11 +16,17 @@
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include "os.h"
#include "taos.h"
......@@ -461,11 +467,11 @@ void *controlThread(void *param) {
}
void *queryQueueThread(void *param) {
return NULL;
}
void *fetchQueueThread(void *param) {
return NULL;
}
......@@ -783,5 +789,4 @@ int main(int argc, char** argv) {
return RUN_ALL_TESTS();
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/scheduler/inc/schedulerInt.h
浏览文件 @ 6e358fd4
......@@ -146,12 +146,15 @@ typedef struct SSchJob {
#define SCH_SET_JOB_TYPE(pAttr, type) (pAttr)->queryJob = ((type) != QUERY_TYPE_MODIFY)
#define SCH_JOB_NEED_FETCH(pAttr) ((pAttr)->queryJob)
#define SCH_JOB_ELOG(param, ...) qError("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__)
#define SCH_JOB_DLOG(param, ...) qDebug("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__)
#define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__)
#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__)
#define SCH_TASK_WLOG(param, ...) qWarn("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__)
#define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__)
#define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__)
#define SCH_TASK_ELOG(param, ...) \
qError("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__)
#define SCH_TASK_DLOG(param, ...) \
qDebug("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__)
#define SCH_TASK_WLOG(param, ...) \
qWarn("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__)
#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
......
source/libs/scheduler/src/scheduler.c
浏览文件 @ 6e358fd4
......@@ -639,7 +639,6 @@ _return:
int32_t schProcessOnDataFetched(SSchJob *job) {
atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0);
tsem_post(&job->rspSem);
}
......@@ -858,11 +857,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
}
atomic_store_ptr(&pJob->res, rsp);
atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows);
if (rsp->completed) {
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED);
SCH_ERR_JRET(schProcessOnDataFetched(pJob));
break;
}
SCH_ERR_JRET(schProcessOnDataFetched(pJob));
......@@ -876,7 +872,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
}
default:
SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask));
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
......@@ -918,8 +913,7 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in
}
pTask = *task;
SCH_TASK_DLOG("rsp msg received, type:%d, code:%x", msgType, rspCode);
SCH_TASK_DLOG("rsp msg received, type:%s, code:%s", TMSG_INFO(msgType), tstrerror(rspCode));
SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode));
......@@ -1469,11 +1463,11 @@ int32_t schedulerConvertDagToTaskList(SQueryDag *pDag, SArray **pTasks) {
pMsg->header.vgId = htonl(tInfo.addr.nodeId);
pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(plan->id.queryId);
pMsg->taskId = htobe64(schGenUUID());
pMsg->sId = schMgmt.sId;
pMsg->queryId = plan->id.queryId;
pMsg->taskId = schGenUUID();
pMsg->taskType = TASK_TYPE_PERSISTENT;
pMsg->contentLen = htonl(msgLen);
pMsg->contentLen = msgLen;
memcpy(pMsg->msg, msg, msgLen);
tInfo.msg = pMsg;
......@@ -1545,8 +1539,8 @@ int32_t scheduleFetchRows(SSchJob *pJob, void **pData) {
if (NULL == pJob || NULL == pData) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
int32_t code = 0;
int32_t code = 0;
atomic_add_fetch_32(&pJob->ref, 1);
int8_t status = SCH_GET_JOB_STATUS(pJob);
......@@ -1592,7 +1586,6 @@ _return:
while (true) {
*pData = atomic_load_ptr(&pJob->res);
if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) {
continue;
}
......@@ -1611,8 +1604,7 @@ _return:
atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
SCH_JOB_DLOG("fetch done, code:%x", code);
SCH_JOB_DLOG("fetch done, code:%s", tstrerror(code));
atomic_sub_fetch_32(&pJob->ref, 1);
SCH_RET(code);
......
source/libs/scheduler/test/schedulerTests.cpp
浏览文件 @ 6e358fd4
......@@ -16,11 +16,7 @@
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "taos.h"
......@@ -30,6 +26,16 @@
#include "scheduler.h"
#include "tep.h"
#include "trpc.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wliteral-suffix"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wreturn-type"
#pragma GCC diagnostic ignored "-Wformat"
#include "schedulerInt.h"
#include "stub.h"
#include "addr_any.h"
......@@ -680,6 +686,4 @@ int main(int argc, char** argv) {
return RUN_ALL_TESTS();
}
#pragma GCC diagnostic pop
\ No newline at end of file
source/libs/sync/src/raft_handle_vote_message.c
浏览文件 @ 6e358fd4
......@@ -37,7 +37,7 @@ int syncRaftHandleVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) {
if (pRespMsg == NULL) {
return 0;
}
syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 ", vote: %d] %s for %d"\
syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 ", vote: %d] %s for %d"
"[logterm: %" PRId64 ", index: %" PRId64 "] at term %" PRId64 "",
pRaft->selfGroupId, pRaft->selfId, lastTerm, lastIndex, pRaft->voteFor,
grant ? "grant" : "reject",
......
source/libs/sync/src/sync.c
浏览文件 @ 6e358fd4
......@@ -23,8 +23,8 @@ SSyncManager* gSyncManager = NULL;
#define SYNC_ACTIVITY_TIMER 5
#define SYNC_SERVER_WORKER 2
static void syncProcessRsp(SRpcMsg *pMsg, SEpSet *pEpSet);
static void syncProcessReqMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
static void syncProcessRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
static void syncProcessReqMsg(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyncCfg);
static int syncInitRpcClient(SSyncManager* syncManager);
......@@ -119,7 +119,7 @@ SSyncNode* syncStart(const SSyncInfo* pInfo) {
return NULL;
}
pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)pInfo->vgId, gSyncManager->syncTimerManager);
pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)((int64_t)pInfo->vgId), gSyncManager->syncTimerManager);
// start raft
pNode->raft.pNode = pNode;
......@@ -176,12 +176,12 @@ int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode) {
}
// process rpc rsp message from other sync server
static void syncProcessRsp(SRpcMsg *pMsg, SEpSet *pEpSet) {
static void syncProcessRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
}
// process rpc message from other sync server
static void syncProcessReqMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
static void syncProcessReqMsg(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
}
......@@ -195,7 +195,7 @@ static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyn
}
assert(pSyncCfg->selfIndex < pSyncCfg->replica && pSyncCfg->selfIndex >= 0);
const SNodeInfo* pNode = &(pSyncCfg->nodeInfo[pSyncCfg->replica]);
char buffer[20] = {'\0'};
char buffer[156] = {'\0'};
snprintf(buffer, sizeof(buffer), "%s:%d", &(pNode->nodeFqdn[0]), pNode->nodePort);
size_t len = strlen(buffer);
void** ppRpcServer = taosHashGet(gSyncManager->rpcServerTable, buffer, len);
......@@ -287,7 +287,7 @@ static void *syncWorkerMain(void *argv) {
}
static void syncNodeTick(void *param, void *tmrId) {
SyncGroupId vgId = (SyncGroupId)param;
SyncGroupId vgId = (SyncGroupId)((int64_t)param);
SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &vgId, sizeof(SyncGroupId*));
if (ppNode == NULL) {
return;
......@@ -298,5 +298,5 @@ static void syncNodeTick(void *param, void *tmrId) {
syncRaftTick(&pNode->raft);
pthread_mutex_unlock(&pNode->mutex);
pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)pNode->vgId, gSyncManager->syncTimerManager);
pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)(int64_t)pNode->vgId, gSyncManager->syncTimerManager);
}
\ No newline at end of file
source/libs/tfs/src/tfs.c
浏览文件 @ 6e358fd4
......@@ -118,7 +118,11 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) {
pDiskId->id = -1;
if (pDiskId->level >= pTfs->nlevel) {
pDiskId->level--;
pDiskId->level = pTfs->nlevel - 1;
}
if (pDiskId->level < 0) {
pDiskId->level = 0;
}
while (pDiskId->level >= 0) {
......@@ -289,7 +293,7 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) {
STfsDisk *pDisk = pTier->disks[id];
snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname);
snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname);
if (taosRenameFile(oaname, naname) != 0) {
if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) {
terrno = TAOS_SYSTEM_ERROR(errno);
fError("failed to rename %s to %s since %s", oaname, naname, terrstr());
return -1;
......
source/libs/tfs/src/tfsTier.c
浏览文件 @ 6e358fd4
......@@ -69,7 +69,7 @@ STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) {
pTier->disks[id] = pDisk;
pTier->ndisk++;
fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id);
fDebug("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id);
return pTier->disks[id];
}
......
source/libs/tfs/test/tfsTest.cpp
浏览文件 @ 6e358fd4
......@@ -246,20 +246,19 @@ TEST_F(TfsTest, 04_File) {
snprintf(fulldir, 128, "%s%s%s", root, TD_DIRSEP, "t3");
EXPECT_STREQ(dir, fulldir);
EXPECT_NE(tfsCopyFile(&f1, &f2), 0);
EXPECT_GT(tfsCopyFile(&f1, &f2), 0);
char af2[128] = {0};
snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2);
EXPECT_EQ(taosDirExist(af2), 0);
tfsRemoveFile(&f2);
EXPECT_NE(taosDirExist(af2), 0);
EXPECT_NE(tfsCopyFile(&f1, &f2), 0);
{
STfsDir *pDir = tfsOpendir(pTfs, "");
STfsDir *pDir = tfsOpendir(pTfs, "t3");
const STfsFile *pf1 = tfsReaddir(pDir);
EXPECT_STREQ(pf1->rname, "t3");
EXPECT_NE(pf1, nullptr);
EXPECT_EQ(pf1->did.id, 0);
EXPECT_EQ(pf1->did.level, 0);
EXPECT_EQ(pf1->pTfs, pTfs);
......@@ -270,6 +269,8 @@ TEST_F(TfsTest, 04_File) {
tfsClosedir(pDir);
}
EXPECT_GT(tfsCopyFile(&f1, &f2), 0);
{
STfsDir *pDir = tfsOpendir(pTfs, "t3");
......@@ -280,7 +281,418 @@ TEST_F(TfsTest, 04_File) {
EXPECT_EQ(pf1->pTfs, pTfs);
const STfsFile *pf2 = tfsReaddir(pDir);
EXPECT_NE(pf2, nullptr);
EXPECT_EQ(pf2->did.id, 0);
EXPECT_EQ(pf2->did.level, 0);
EXPECT_EQ(pf2->pTfs, pTfs);
const STfsFile *pf3 = tfsReaddir(pDir);
EXPECT_EQ(pf3, nullptr);
tfsClosedir(pDir);
}
}
tfsClose(pTfs);
}
TEST_F(TfsTest, 05_MultiDisk) {
int32_t code = 0;
const char *root00 = "/tmp/tfsTest00";
const char *root01 = "/tmp/tfsTest01";
const char *root10 = "/tmp/tfsTest10";
const char *root11 = "/tmp/tfsTest11";
const char *root12 = "/tmp/tfsTest12";
const char *root20 = "/tmp/tfsTest20";
const char *root21 = "/tmp/tfsTest21";
const char *root22 = "/tmp/tfsTest22";
const char *root23 = "/tmp/tfsTest23";
SDiskCfg dCfg[9] = {0};
tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN);
dCfg[0].level = 0;
dCfg[0].primary = 0;
tstrncpy(dCfg[1].dir, root00, TSDB_FILENAME_LEN);
dCfg[1].level = 0;
dCfg[1].primary = 0;
tstrncpy(dCfg[2].dir, root20, TSDB_FILENAME_LEN);
dCfg[2].level = 2;
dCfg[2].primary = 0;
tstrncpy(dCfg[3].dir, root21, TSDB_FILENAME_LEN);
dCfg[3].level = 2;
dCfg[3].primary = 0;
tstrncpy(dCfg[4].dir, root22, TSDB_FILENAME_LEN);
dCfg[4].level = 2;
dCfg[4].primary = 0;
tstrncpy(dCfg[5].dir, root23, TSDB_FILENAME_LEN);
dCfg[5].level = 2;
dCfg[5].primary = 0;
tstrncpy(dCfg[6].dir, root10, TSDB_FILENAME_LEN);
dCfg[6].level = 1;
dCfg[6].primary = 0;
tstrncpy(dCfg[7].dir, root11, TSDB_FILENAME_LEN);
dCfg[7].level = 1;
dCfg[7].primary = 0;
tstrncpy(dCfg[8].dir, root12, TSDB_FILENAME_LEN);
dCfg[8].level = 1;
dCfg[8].primary = 0;
taosRemoveDir(root00);
taosRemoveDir(root01);
taosRemoveDir(root10);
taosRemoveDir(root11);
taosRemoveDir(root12);
taosRemoveDir(root20);
taosRemoveDir(root21);
taosRemoveDir(root22);
taosRemoveDir(root23);
taosMkDir(root00);
taosMkDir(root01);
taosMkDir(root10);
taosMkDir(root11);
taosMkDir(root12);
taosMkDir(root20);
taosMkDir(root21);
taosMkDir(root22);
taosMkDir(root23);
STfs *pTfs = tfsOpen(dCfg, 9);
ASSERT_EQ(pTfs, nullptr);
dCfg[0].primary = 1;
dCfg[1].primary = 1;
pTfs = tfsOpen(dCfg, 9);
ASSERT_EQ(pTfs, nullptr);
dCfg[0].primary = 0;
dCfg[1].primary = 1;
pTfs = tfsOpen(dCfg, 9);
ASSERT_NE(pTfs, nullptr);
tfsUpdateSize(pTfs);
SDiskSize size = tfsGetSize(pTfs);
EXPECT_GT(size.avail, 0);
EXPECT_GT(size.used, 0);
EXPECT_GT(size.total, size.avail);
EXPECT_GT(size.total, size.used);
//------------- AllocDisk -----------------//
{
const char *path = NULL;
SDiskID did;
did.id = 0;
did.level = 0;
code = tfsAllocDisk(pTfs, 0, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 0);
EXPECT_EQ(did.level, 0);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root00);
code = tfsAllocDisk(pTfs, 0, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 1);
EXPECT_EQ(did.level, 0);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root01);
code = tfsAllocDisk(pTfs, 0, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 0);
EXPECT_EQ(did.level, 0);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root00);
code = tfsAllocDisk(pTfs, 0, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 1);
EXPECT_EQ(did.level, 0);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root01);
code = tfsAllocDisk(pTfs, 0, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 0);
EXPECT_EQ(did.level, 0);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root00);
code = tfsAllocDisk(pTfs, 0, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 1);
EXPECT_EQ(did.level, 0);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root01);
code = tfsAllocDisk(pTfs, 1, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 0);
EXPECT_EQ(did.level, 1);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root10);
code = tfsAllocDisk(pTfs, 1, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 1);
EXPECT_EQ(did.level, 1);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root11);
code = tfsAllocDisk(pTfs, 1, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 2);
EXPECT_EQ(did.level, 1);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root12);
code = tfsAllocDisk(pTfs, 1, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 0);
EXPECT_EQ(did.level, 1);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root10);
code = tfsAllocDisk(pTfs, 2, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 0);
EXPECT_EQ(did.level, 2);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root20);
code = tfsAllocDisk(pTfs, 2, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 1);
EXPECT_EQ(did.level, 2);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root21);
code = tfsAllocDisk(pTfs, 2, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 2);
EXPECT_EQ(did.level, 2);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root22);
code = tfsAllocDisk(pTfs, 2, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 3);
EXPECT_EQ(did.level, 2);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root23);
code = tfsAllocDisk(pTfs, 2, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 0);
EXPECT_EQ(did.level, 2);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root20);
code = tfsAllocDisk(pTfs, 3, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 1);
EXPECT_EQ(did.level, 2);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root21);
code = tfsAllocDisk(pTfs, 4, &did);
EXPECT_EQ(code, 0);
EXPECT_EQ(did.id, 2);
EXPECT_EQ(did.level, 2);
path = tfsGetDiskPath(pTfs, did);
EXPECT_STREQ(path, root22);
const char *primary = tfsGetPrimaryPath(pTfs);
EXPECT_STREQ(primary, root00);
}
//------------- Dir -----------------//
{
char p1[] = "p1";
char ap00[128] = {0};
snprintf(ap00, 128, "%s%s%s", root00, TD_DIRSEP, p1);
char ap01[128] = {0};
snprintf(ap01, 128, "%s%s%s", root01, TD_DIRSEP, p1);
char ap10[128] = {0};
snprintf(ap10, 128, "%s%s%s", root10, TD_DIRSEP, p1);
char ap11[128] = {0};
snprintf(ap11, 128, "%s%s%s", root11, TD_DIRSEP, p1);
char ap12[128] = {0};
snprintf(ap12, 128, "%s%s%s", root12, TD_DIRSEP, p1);
char ap20[128] = {0};
snprintf(ap20, 128, "%s%s%s", root20, TD_DIRSEP, p1);
char ap21[128] = {0};
snprintf(ap21, 128, "%s%s%s", root21, TD_DIRSEP, p1);
char ap22[128] = {0};
snprintf(ap22, 128, "%s%s%s", root22, TD_DIRSEP, p1);
char ap23[128] = {0};
snprintf(ap23, 128, "%s%s%s", root23, TD_DIRSEP, p1);
EXPECT_NE(taosDirExist(ap00), 0);
EXPECT_NE(taosDirExist(ap01), 0);
EXPECT_NE(taosDirExist(ap10), 0);
EXPECT_NE(taosDirExist(ap11), 0);
EXPECT_NE(taosDirExist(ap12), 0);
EXPECT_NE(taosDirExist(ap20), 0);
EXPECT_NE(taosDirExist(ap21), 0);
EXPECT_NE(taosDirExist(ap22), 0);
EXPECT_NE(taosDirExist(ap23), 0);
EXPECT_EQ(tfsMkdir(pTfs, p1), 0);
EXPECT_EQ(taosDirExist(ap00), 0);
EXPECT_EQ(taosDirExist(ap01), 0);
EXPECT_EQ(taosDirExist(ap10), 0);
EXPECT_EQ(taosDirExist(ap11), 0);
EXPECT_EQ(taosDirExist(ap12), 0);
EXPECT_EQ(taosDirExist(ap20), 0);
EXPECT_EQ(taosDirExist(ap21), 0);
EXPECT_EQ(taosDirExist(ap22), 0);
EXPECT_EQ(taosDirExist(ap23), 0);
EXPECT_EQ(tfsRmdir(pTfs, p1), 0);
EXPECT_NE(taosDirExist(ap00), 0);
EXPECT_NE(taosDirExist(ap01), 0);
EXPECT_NE(taosDirExist(ap10), 0);
EXPECT_NE(taosDirExist(ap11), 0);
EXPECT_NE(taosDirExist(ap12), 0);
EXPECT_NE(taosDirExist(ap20), 0);
EXPECT_NE(taosDirExist(ap21), 0);
EXPECT_NE(taosDirExist(ap22), 0);
EXPECT_NE(taosDirExist(ap23), 0);
char p2[] = "p2";
char _ap21[128] = {0};
snprintf(_ap21, 128, "%s%s%s", root21, TD_DIRSEP, p2);
SDiskID did = {0};
did.level = 2;
did.id = 1;
EXPECT_NE(taosDirExist(_ap21), 0);
EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0);
EXPECT_EQ(taosDirExist(_ap21), 0);
char p3[] = "p3/p2/p1/p0";
char _ap12[128] = {0};
snprintf(_ap12, 128, "%s%s%s", root12, TD_DIRSEP, p3);
did.level = 1;
did.id = 2;
EXPECT_NE(taosDirExist(_ap12), 0);
EXPECT_NE(tfsMkdir(pTfs, p3), 0);
EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0);
EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0);
EXPECT_EQ(taosDirExist(_ap12), 0);
EXPECT_EQ(tfsRmdir(pTfs, p3), 0);
EXPECT_NE(taosDirExist(_ap12), 0);
char p45[] = "p5";
char p44[] = "p4";
char p4[] = "p4/p2/p1/p0";
char _ap22[128] = {0};
snprintf(_ap22, 128, "%s%s%s", root22, TD_DIRSEP, p4);
did.level = 2;
did.id = 2;
EXPECT_NE(taosDirExist(_ap22), 0);
EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0);
EXPECT_EQ(taosDirExist(_ap22), 0);
EXPECT_EQ(tfsRename(pTfs, p44, p45), 0);
EXPECT_EQ(tfsRmdir(pTfs, p4), 0);
EXPECT_NE(taosDirExist(_ap22), 0);
}
//------------- File -----------------//
{
STfsFile file0;
STfsFile file1;
STfsFile file2;
STfsFile file3;
STfsFile file4;
SDiskID did0 = {0};
SDiskID did1 = {0};
SDiskID did2 = {0};
SDiskID did3 = {0};
SDiskID did4 = {0};
did3.id = 1;
did4.level = 1;
tfsInitFile(pTfs, &file0, did0, "fname");
tfsInitFile(pTfs, &file1, did1, "fname");
tfsInitFile(pTfs, &file2, did2, "fnamex");
tfsInitFile(pTfs, &file3, did3, "fname");
tfsInitFile(pTfs, &file4, did4, "fname");
EXPECT_TRUE(tfsIsSameFile(&file0, &file1));
EXPECT_FALSE(tfsIsSameFile(&file0, &file2));
EXPECT_FALSE(tfsIsSameFile(&file0, &file3));
EXPECT_FALSE(tfsIsSameFile(&file0, &file4));
{
char n1[] = "t3/t1.json";
char n2[] = "t3/t2.json";
STfsFile f1 = {0};
STfsFile f2 = {0};
SDiskID did;
did1.level = 1;
did1.id = 2;
did2.level = 2;
did2.id = 3;
tfsInitFile(pTfs, &f1, did1, n1);
tfsInitFile(pTfs, &f2, did2, n2);
EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
FILE *fp = fopen(f1.aname, "w");
ASSERT_NE(fp, nullptr);
fwrite("12345678", 1, 5, fp);
fclose(fp);
char base[128] = {0};
tfsBasename(&f1, base);
char dir[128] = {0};
tfsDirname(&f1, dir);
EXPECT_STREQ(base, "t1.json");
char fulldir[128];
snprintf(fulldir, 128, "%s%s%s", root12, TD_DIRSEP, "t3");
EXPECT_STREQ(dir, fulldir);
EXPECT_GT(tfsCopyFile(&f1, &f2), 0);
char af2[128] = {0};
snprintf(af2, 128, "%s%s%s", root23, TD_DIRSEP, n2);
EXPECT_EQ(taosDirExist(af2), 0);
tfsRemoveFile(&f2);
{
STfsDir *pDir = tfsOpendir(pTfs, "t3");
const STfsFile *pf1 = tfsReaddir(pDir);
EXPECT_NE(pf1, nullptr);
EXPECT_EQ(pf1->did.level, 1);
EXPECT_EQ(pf1->did.id, 2);
EXPECT_EQ(pf1->pTfs, pTfs);
const STfsFile *pf2 = tfsReaddir(pDir);
EXPECT_EQ(pf2, nullptr);
tfsClosedir(pDir);
}
EXPECT_NE(taosDirExist(af2), 0);
EXPECT_GT(tfsCopyFile(&f1, &f2), 0);
{
STfsDir *pDir = tfsOpendir(pTfs, "t3");
const STfsFile *pf1 = tfsReaddir(pDir);
EXPECT_NE(pf1, nullptr);
EXPECT_GT(pf1->did.level, 0);
EXPECT_GT(pf1->did.id, 0);
EXPECT_EQ(pf1->pTfs, pTfs);
const STfsFile *pf2 = tfsReaddir(pDir);
EXPECT_NE(pf1, nullptr);
EXPECT_GT(pf1->did.level, 0);
EXPECT_GT(pf1->did.id, 0);
EXPECT_EQ(pf1->pTfs, pTfs);
const STfsFile *pf3 = tfsReaddir(pDir);
EXPECT_EQ(pf3, nullptr);
......@@ -288,6 +700,7 @@ TEST_F(TfsTest, 04_File) {
tfsClosedir(pDir);
}
}
}
tfsClose(pTfs);
}
\ No newline at end of file
source/libs/transport/src/rpcMain.c
浏览文件 @ 6e358fd4
......@@ -752,8 +752,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
}
taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid,
hashstr);
tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s spi:%d", pRpc->label, pConn, pConn->linkUid, sid,
hashstr, pConn->spi);
}
return pConn;
......@@ -1612,7 +1612,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) {
}
}
} else {
tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi);
tError("%s, auth spi:%d not matched with received:%d %p", pConn->info, pConn->spi, pHead->spi, pConn);
code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED;
}
......
source/libs/transport/src/trans.c
浏览文件 @ 6e358fd4
......@@ -32,6 +32,7 @@ void* rpcOpen(const SRpcInit* pInit) {
pRpc->cfp = pInit->cfp;
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
pRpc->connType = pInit->connType;
pRpc->idleTime = pInit->idleTime;
pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc);
return pRpc;
......
source/libs/transport/src/transCli.c
浏览文件 @ 6e358fd4
......@@ -17,10 +17,13 @@
#include "transComm.h"
#define CONN_PERSIST_TIME(para) (para * 1000 * 10)
typedef struct SCliConn {
uv_connect_t connReq;
uv_stream_t* stream;
uv_write_t* writeReq;
void* hostThrd;
SConnBuffer readBuf;
void* data;
queue conn;
......@@ -41,11 +44,11 @@ typedef struct SCliThrdObj {
uv_loop_t* loop;
uv_async_t* cliAsync; //
uv_timer_t* pTimer;
void* cache; // conn pool
void* pool; // conn pool
queue msg;
pthread_mutex_t msgMtx;
uint64_t nextTimeout; // next timeout
void* shandle; //
void* pTransInst; //
} SCliThrdObj;
......@@ -62,15 +65,15 @@ typedef struct SConnList {
// conn pool
// add expire timeout and capacity limit
static void* connCacheCreate(int size);
static void* connCacheDestroy(void* cache);
static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port);
static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn);
static void* creatConnPool(int size);
static void* destroyConnPool(void* pool);
static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port);
static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn);
// register timer in each thread to clear expire conn
static void clientTimeoutCb(uv_timer_t* handle);
// process data read from server, auth/decompress etc
static void clientProcessData(SCliConn* conn);
// process data read from server, auth/decompress etc later
static void clientHandleResp(SCliConn* conn);
// check whether already read complete packet from server
static bool clientReadComplete(SConnBuffer* pBuf);
// alloc buf for read
......@@ -83,78 +86,94 @@ static void clientWriteCb(uv_write_t* req, int status);
static void clientConnCb(uv_connect_t* req, int status);
static void clientAsyncCb(uv_async_t* handle);
static void clientDestroy(uv_handle_t* handle);
static void clientConnDestroy(SCliConn* pConn);
static void clientConnDestroy(SCliConn* pConn, bool clear /*clear tcp handle or not*/);
static void clientMsgDestroy(SCliMsg* pMsg);
// handle req from app
static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
// thread obj
static SCliThrdObj* createThrdObj();
static void destroyThrdObj(SCliThrdObj* pThrd);
// thread
static void* clientThread(void* arg);
static void clientProcessData(SCliConn* conn) {
static void clientHandleResp(SCliConn* conn) {
STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx;
SRpcInfo* pRpc = pCtx->ahandle;
SRpcInfo* pRpc = pCtx->pRpc;
SRpcMsg rpcMsg;
rpcMsg.pCont = conn->readBuf.buf;
rpcMsg.contLen = conn->readBuf.len;
rpcMsg.ahandle = pCtx->ahandle;
(pRpc->cfp)(NULL, &rpcMsg, NULL);
SCliThrdObj* pThrd = conn->hostThrd;
addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn);
if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) {
uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
}
free(pCtx->ip);
free(pCtx);
// impl
}
static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void clientTimeoutCb(uv_timer_t* handle) {
SCliThrdObj* pThrd = handle->data;
SRpcInfo* pRpc = pThrd->shandle;
SRpcInfo* pRpc = pThrd->pTransInst;
int64_t currentTime = pThrd->nextTimeout;
tDebug("timeout, try to remove expire conn from conn pool");
SConnList* p = taosHashIterate((SHashObj*)pThrd->cache, NULL);
SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL);
while (p != NULL) {
while (!QUEUE_IS_EMPTY(&p->conn)) {
queue* h = QUEUE_HEAD(&p->conn);
SCliConn* c = QUEUE_DATA(h, SCliConn, conn);
if (c->expireTime < currentTime) {
QUEUE_REMOVE(h);
clientConnDestroy(c);
// uv_stream_t stm = *(c->stream);
// uv_close((uv_handle_t*)&stm, clientDestroy);
clientConnDestroy(c, true);
} else {
break;
}
}
p = taosHashIterate((SHashObj*)pThrd->cache, p);
p = taosHashIterate((SHashObj*)pThrd->pool, p);
}
pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10;
uv_timer_start(handle, clientTimeoutCb, pRpc->idleTime * 10, 0);
pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
uv_timer_start(handle, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
}
static void* connCacheCreate(int size) {
SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
return false;
static void* creatConnPool(int size) {
// thread local, no lock
return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
}
static void* connCacheDestroy(void* cache) {
SConnList* connList = taosHashIterate((SHashObj*)cache, NULL);
static void* destroyConnPool(void* pool) {
SConnList* connList = taosHashIterate((SHashObj*)pool, NULL);
while (connList != NULL) {
while (!QUEUE_IS_EMPTY(&connList->conn)) {
queue* h = QUEUE_HEAD(&connList->conn);
QUEUE_REMOVE(h);
SCliConn* c = QUEUE_DATA(h, SCliConn, conn);
clientConnDestroy(c);
clientConnDestroy(c, true);
}
connList = taosHashIterate((SHashObj*)cache, connList);
connList = taosHashIterate((SHashObj*)pool, connList);
}
taosHashClear(cache);
taosHashClear(pool);
}
static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) {
static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
char key[128] = {0};
tstrncpy(key, ip, strlen(ip));
tstrncpy(key + strlen(key), (char*)(&port), sizeof(port));
SHashObj* pCache = cache;
SConnList* plist = taosHashGet(pCache, key, strlen(key));
SHashObj* pPool = pool;
SConnList* plist = taosHashGet(pPool, key, strlen(key));
if (plist == NULL) {
SConnList list;
plist = &list;
taosHashPut(pPool, key, strlen(key), (void*)&list, sizeof(list));
plist = taosHashGet(pPool, key, strlen(key));
QUEUE_INIT(&plist->conn);
taosHashPut(pCache, key, strlen(key), plist, sizeof(*plist));
}
if (QUEUE_IS_EMPTY(&plist->conn)) {
......@@ -164,15 +183,14 @@ static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) {
QUEUE_REMOVE(h);
return QUEUE_DATA(h, SCliConn, conn);
}
static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) {
static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) {
char key[128] = {0};
tstrncpy(key, ip, strlen(ip));
tstrncpy(key + strlen(key), (char*)(&port), sizeof(port));
STransConnCtx* ctx = ((SCliMsg*)conn->data)->ctx;
SRpcInfo* pRpc = ctx->pRpc;
conn->expireTime = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10;
SConnList* plist = taosHashGet((SHashObj*)cache, key, strlen(key));
SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst;
conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
// list already create before
assert(plist != NULL);
QUEUE_PUSH(&plist->conn, &conn->conn);
......@@ -200,10 +218,11 @@ static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size,
SCliConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
if (pBuf->cap == 0) {
pBuf->buf = (char*)calloc(CAPACITY, sizeof(char));
pBuf->buf = (char*)calloc(1, CAPACITY * sizeof(char));
pBuf->len = 0;
pBuf->cap = CAPACITY;
pBuf->left = -1;
buf->base = pBuf->buf;
buf->len = CAPACITY;
} else {
......@@ -213,7 +232,7 @@ static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size,
pBuf->buf = realloc(pBuf->buf, pBuf->cap);
} else if (pBuf->len + pBuf->left > pBuf->cap) {
pBuf->cap = pBuf->len + pBuf->left;
pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left);
pBuf->buf = realloc(pBuf->buf, pBuf->cap);
}
}
buf->base = pBuf->buf + pBuf->len;
......@@ -227,29 +246,38 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
if (nread > 0) {
pBuf->len += nread;
if (clientReadComplete(pBuf)) {
tDebug("alread read complete pack");
clientProcessData(conn);
tDebug("alread read complete");
clientHandleResp(conn);
} else {
tDebug("read halp packet, continue to read");
tDebug("read half packet, continue to read");
}
return;
}
assert(nread <= 0);
if (nread == 0) {
return;
}
if (nread != UV_EOF) {
tDebug("Read error %s\n", uv_err_name(nread));
tDebug("read error %s", uv_err_name(nread));
}
//
uv_close((uv_handle_t*)handle, clientDestroy);
// tDebug("Read error %s\n", uv_err_name(nread));
// uv_close((uv_handle_t*)handle, clientDestroy);
}
static void clientConnDestroy(SCliConn* conn) {
// impl later
//
static void clientConnDestroy(SCliConn* conn, bool clear) {
tDebug("conn %p destroy", conn);
if (clear) {
uv_close((uv_handle_t*)conn->stream, NULL);
}
free(conn->stream);
free(conn->readBuf.buf);
free(conn->writeReq);
free(conn);
}
static void clientDestroy(uv_handle_t* handle) {
SCliConn* conn = handle->data;
QUEUE_REMOVE(&conn->conn);
clientConnDestroy(conn);
// QUEUE_REMOVE(&conn->conn);
clientConnDestroy(conn, false);
}
static void clientWriteCb(uv_write_t* req, int status) {
......@@ -257,10 +285,16 @@ static void clientWriteCb(uv_write_t* req, int status) {
if (status == 0) {
tDebug("data already was written on stream");
} else {
uv_close((uv_handle_t*)pConn->stream, clientDestroy);
tError("failed to write: %s", uv_err_name(status));
clientConnDestroy(pConn, true);
return;
}
SCliThrdObj* pThrd = pConn->hostThrd;
if (pConn->stream == NULL) {
pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t));
uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream);
pConn->stream->data = pConn;
}
uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb);
// impl later
}
......@@ -271,34 +305,36 @@ static void clientWrite(SCliConn* pConn) {
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
int msgLen = transMsgLenFromCont(pMsg->contLen);
char* msg = (char*)(pHead);
uv_buf_t wb = uv_buf_init(msg, msgLen);
pHead->msgType = pMsg->msgType;
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
tDebug("data write out, msgType : %d, len: %d", pHead->msgType, msgLen);
uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb);
}
static void clientConnCb(uv_connect_t* req, int status) {
// impl later
SCliConn* pConn = req->data;
if (status != 0) {
tError("failed to connect %s", uv_err_name(status));
clientConnDestroy(pConn);
return;
}
SCliMsg* pMsg = pConn->data;
STransConnCtx* pCtx = ((SCliMsg*)(pConn->data))->ctx;
SRpcMsg rpcMsg;
rpcMsg.ahandle = pCtx->ahandle;
STransConnCtx* pCtx = pMsg->ctx;
SRpcInfo* pRpc = pCtx->pRpc;
if (status != 0) {
// tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status));
tError("failed to connect server, errmsg: %s", uv_strerror(status));
// call user fp later
tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status));
SRpcInfo* pRpc = pMsg->ctx->pRpc;
SRpcMsg rpcMsg;
rpcMsg.ahandle = pCtx->ahandle;
// SRpcInfo* pRpc = pMsg->ctx->pRpc;
(pRpc->cfp)(NULL, &rpcMsg, NULL);
uv_close((uv_handle_t*)req->handle, clientDestroy);
clientConnDestroy(pConn, true);
// uv_close((uv_handle_t*)req->handle, clientDestroy);
return;
}
assert(pConn->stream == req->handle);
clientWrite(pConn);
}
......@@ -310,22 +346,32 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
et = taosGetTimestampUs();
STransConnCtx* pCtx = pMsg->ctx;
SCliConn* conn = getConnFromCache(pThrd->cache, pCtx->ip, pCtx->port);
SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
if (conn != NULL) {
// impl later
conn->data = pMsg;
conn->writeReq->data = conn;
conn->readBuf.len = 0;
memset(conn->readBuf.buf, 0, conn->readBuf.cap);
conn->readBuf.left = -1;
clientWrite(conn);
} else {
SCliConn* conn = calloc(1, sizeof(SCliConn));
// read/write stream handle
conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t));
uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream));
conn->stream->data = conn;
// write req handle
conn->writeReq = malloc(sizeof(uv_write_t));
conn->writeReq->data = conn;
QUEUE_INIT(&conn->conn);
conn->connReq.data = conn;
conn->data = pMsg;
conn->hostThrd = pThrd;
struct sockaddr_in addr;
uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr);
......@@ -351,31 +397,47 @@ static void clientAsyncCb(uv_async_t* handle) {
SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q);
clientHandleReq(pMsg, pThrd);
count++;
if (count >= 2) {
tError("send batch size: %d", count);
}
if (count >= 2) {
tDebug("already process batch size: %d", count);
}
}
static void* clientThread(void* arg) {
SCliThrdObj* pThrd = (SCliThrdObj*)arg;
SRpcInfo* pRpc = pThrd->shandle;
pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10;
uv_timer_start(pThrd->pTimer, clientTimeoutCb, pRpc->idleTime * 10, 0);
uv_run(pThrd->loop, UV_RUN_DEFAULT);
}
void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) {
SClientObj* cli = calloc(1, sizeof(SClientObj));
SRpcInfo* pRpc = shandle;
memcpy(cli->label, label, strlen(label));
cli->numOfThreads = numOfThreads;
cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*));
for (int i = 0; i < cli->numOfThreads; i++) {
SCliThrdObj* pThrd = createThrdObj();
pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
pThrd->pTransInst = shandle;
int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd));
if (err == 0) {
tDebug("sucess to create tranport-client thread %d", i);
}
cli->pThreadObj[i] = pThrd;
}
return cli;
}
static void clientMsgDestroy(SCliMsg* pMsg) {
// impl later
free(pMsg);
}
static SCliThrdObj* createThrdObj() {
SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj));
QUEUE_INIT(&pThrd->msg);
pthread_mutex_init(&pThrd->msgMtx, NULL);
pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
uv_loop_init(pThrd->loop);
......@@ -385,31 +447,27 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
pThrd->pTimer = malloc(sizeof(uv_timer_t));
uv_timer_init(pThrd->loop, pThrd->pTimer);
pThrd->pTimer->data = pThrd;
pThrd->shandle = shandle;
int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd));
if (err == 0) {
tDebug("sucess to create tranport-client thread %d", i);
}
cli->pThreadObj[i] = pThrd;
}
return cli;
pThrd->pool = creatConnPool(1);
return pThrd;
}
static void clientMsgDestroy(SCliMsg* pMsg) {
// impl later
free(pMsg);
}
void taosCloseClient(void* arg) {
// impl later
SClientObj* cli = arg;
for (int i = 0; i < cli->numOfThreads; i++) {
SCliThrdObj* pThrd = cli->pThreadObj[i];
static void destroyThrdObj(SCliThrdObj* pThrd) {
if (pThrd == NULL) {
return;
}
pthread_join(pThrd->thread, NULL);
pthread_mutex_destroy(&pThrd->msgMtx);
free(pThrd->cliAsync);
free(pThrd->loop);
free(pThrd);
}
//
void taosCloseClient(void* arg) {
// impl later
SClientObj* cli = arg;
for (int i = 0; i < cli->numOfThreads; i++) {
destroyThrdObj(cli->pThreadObj[i]);
}
free(cli->pThreadObj);
free(cli);
......
source/libs/transport/src/transSrv.c
浏览文件 @ 6e358fd4
......@@ -14,8 +14,8 @@
*/
#ifdef USE_UV
#include "transComm.h"
#include "transComm.h"
typedef struct SConn {
uv_tcp_t* pTcp;
uv_write_t* pWriter;
......@@ -26,11 +26,13 @@ typedef struct SConn {
int ref;
int persist; // persist connection or not
SConnBuffer connBuf; // read buf,
SConnBuffer writeBuf; // write buf
int count;
void* shandle; // rpc init
int inType;
void* pTransInst; // rpc init
void* ahandle; //
void* hostThrd;
SRpcMsg sendMsg;
// del later
char secured;
int spi;
......@@ -48,7 +50,7 @@ typedef struct SWorkThrdObj {
uv_async_t* workerAsync; //
queue conn;
pthread_mutex_t connMtx;
void* shandle;
void* pTransInst;
} SWorkThrdObj;
typedef struct SServerObj {
......@@ -66,7 +68,7 @@ typedef struct SServerObj {
static const char* notify = "a";
// refactor later
static int rpcAddAuthPart(SConn* pConn, char* msg, int msgLen);
static int transAddAuthPart(SConn* pConn, char* msg, int msgLen);
static int uvAuthMsg(SConn* pConn, char* msg, int msgLen);
......@@ -75,21 +77,28 @@ static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_b
static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
static void uvOnTimeoutCb(uv_timer_t* handle);
static void uvOnWriteCb(uv_write_t* req, int status);
static void uvOnPipeWriteCb(uv_write_t* req, int status);
static void uvOnAcceptCb(uv_stream_t* stream, int status);
static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf);
static void uvWorkerAsyncCb(uv_async_t* handle);
// already read complete packet
static void uvPrepareSendData(SConn* conn, uv_buf_t* wb);
// check whether already read complete packet
static bool readComplete(SConnBuffer* buf);
static SConn* createConn();
static void destroyConn(SConn* conn, bool clear /*clear handle or not*/);
static SConn* connCreate();
static void connDestroy(SConn* conn);
static void uvConnDestroy(uv_handle_t* handle);
static void uvDestroyConn(uv_handle_t* handle);
// server worke thread
// server and worker thread
static void* workerThread(void* arg);
static void* acceptThread(void* arg);
// add handle loop
static bool addHandleToWorkloop(void* arg);
static bool addHandleToAcceptloop(void* arg);
void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
/*
* formate of data buffer:
......@@ -135,25 +144,28 @@ static bool readComplete(SConnBuffer* data) {
if (msgLen > data->len) {
data->left = msgLen - data->len;
return false;
} else {
} else if (msgLen == data->len) {
return true;
} else if (msgLen < data->len) {
return false;
// handle other packet later
}
} else {
return false;
}
}
static void uvDoProcess(SRecvInfo* pRecv) {
// impl later
STransMsgHead* pHead = (STransMsgHead*)pRecv->msg;
SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle;
SConn* pConn = pRecv->thandle;
tDump(pRecv->msg, pRecv->msgLen);
terrno = 0;
// SRpcReqContext* pContest;
// do auth and check
}
// static void uvDoProcess(SRecvInfo* pRecv) {
// // impl later
// STransMsgHead* pHead = (STransMsgHead*)pRecv->msg;
// SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle;
// SConn* pConn = pRecv->thandle;
// tDump(pRecv->msg, pRecv->msgLen);
// terrno = 0;
// // SRpcReqContext* pContest;
//
// // do auth and check
//}
static int uvAuthMsg(SConn* pConn, char* msg, int len) {
STransMsgHead* pHead = (STransMsgHead*)msg;
......@@ -222,12 +234,13 @@ static void uvProcessData(SConn* pConn) {
p->msgLen = pBuf->len;
p->ip = 0;
p->port = 0;
p->shandle = pConn->shandle; //
p->shandle = pConn->pTransInst; //
p->thandle = pConn;
p->chandle = NULL;
//
STransMsgHead* pHead = (STransMsgHead*)p->msg;
pConn->inType = pHead->msgType;
assert(transIsReq(pHead->msgType));
SRpcInfo* pRpc = (SRpcInfo*)p->shandle;
......@@ -247,7 +260,9 @@ static void uvProcessData(SConn* pConn) {
// add compress later
// pHead = rpcDecompressRpcMsg(pHead);
} else {
pHead->msgLen = htonl(pHead->msgLen);
// impl later
//
}
rpcMsg.contLen = transContLenFromMsg(pHead->msgLen);
rpcMsg.pCont = pHead->content;
......@@ -257,7 +272,7 @@ static void uvProcessData(SConn* pConn) {
rpcMsg.handle = pConn;
(*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime, 0);
// uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
// auth
// validate msg type
}
......@@ -268,6 +283,7 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
SConnBuffer* pBuf = &conn->connBuf;
if (nread > 0) {
pBuf->len += nread;
tDebug("on read %p, total read: %d, current read: %d", cli, pBuf->len, (int)nread);
if (readComplete(pBuf)) {
tDebug("alread read complete packet");
uvProcessData(conn);
......@@ -276,10 +292,12 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
}
return;
}
if (nread == 0) {
return;
}
if (nread != UV_EOF) {
tDebug("Read error %s\n", uv_err_name(nread));
tDebug("read error %s", uv_err_name(nread));
}
uv_close((uv_handle_t*)cli, uvConnDestroy);
}
void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
buf->base = malloc(sizeof(char));
......@@ -293,16 +311,49 @@ void uvOnTimeoutCb(uv_timer_t* handle) {
void uvOnWriteCb(uv_write_t* req, int status) {
SConn* conn = req->data;
SConnBuffer* buf = &conn->connBuf;
buf->len = 0;
memset(buf->buf, 0, buf->cap);
buf->left = -1;
if (status == 0) {
tDebug("data already was written on stream");
} else {
connDestroy(conn);
tDebug("failed to write data, %s", uv_err_name(status));
destroyConn(conn, true);
}
// opt
}
static void uvOnPipeWriteCb(uv_write_t* req, int status) {
if (status == 0) {
tDebug("success to dispatch conn to work thread");
} else {
tError("fail to dispatch conn to work thread");
}
}
static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) {
// impl later;
tDebug("prepare to send back");
SRpcMsg* pMsg = &conn->sendMsg;
if (pMsg->pCont == 0) {
pMsg->pCont = (void*)rpcMallocCont(0);
pMsg->contLen = 0;
}
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
pHead->msgType = conn->inType + 1;
// add more info
char* msg = (char*)pHead;
int32_t len = transMsgLenFromCont(pMsg->contLen);
if (transCompressMsg(msg, len, NULL)) {
// impl later
}
pHead->msgLen = htonl(len);
wb->base = msg;
wb->len = len;
}
void uvWorkerAsyncCb(uv_async_t* handle) {
SWorkThrdObj* pThrd = container_of(handle, SWorkThrdObj, workerAsync);
SWorkThrdObj* pThrd = handle->data;
SConn* conn = NULL;
queue wq;
// batch process to avoid to lock/unlock frequently
......@@ -318,8 +369,8 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
tError("except occurred, do nothing");
return;
}
uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len);
uv_buf_t wb;
uvPrepareSendData(conn, &wb);
uv_timer_stop(conn->pTimer);
uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb);
......@@ -341,10 +392,12 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) {
uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify));
pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThreads;
tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx);
uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb);
uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb);
} else {
uv_close((uv_handle_t*)cli, NULL);
free(cli);
}
}
void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
......@@ -354,7 +407,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
tError("read error %s", uv_err_name(nread));
}
// TODO(log other failure reason)
uv_close((uv_handle_t*)q, NULL);
// uv_close((uv_handle_t*)q, NULL);
return;
}
// free memory allocated by
......@@ -373,8 +426,8 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
uv_handle_type pending = uv_pipe_pending_type(pipe);
assert(pending == UV_TCP);
SConn* pConn = connCreate();
pConn->shandle = pThrd->shandle;
SConn* pConn = createConn();
pConn->pTransInst = pThrd->pTransInst;
/* init conn timer*/
pConn->pTimer = malloc(sizeof(uv_timer_t));
uv_timer_init(pThrd->loop, pConn->pTimer);
......@@ -398,34 +451,25 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
tDebug("new connection created: %d", fd);
uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb);
} else {
connDestroy(pConn);
tDebug("failed to create new connection");
destroyConn(pConn, true);
}
}
void* acceptThread(void* arg) {
// opt
SServerObj* srv = (SServerObj*)arg;
uv_tcp_init(srv->loop, &srv->server);
struct sockaddr_in bind_addr;
uv_ip4_addr("0.0.0.0", srv->port, &bind_addr);
uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0);
int err = 0;
if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) {
tError("Listen error %s\n", uv_err_name(err));
return NULL;
}
uv_run(srv->loop, UV_RUN_DEFAULT);
}
void* workerThread(void* arg) {
SWorkThrdObj* pThrd = (SWorkThrdObj*)arg;
static bool addHandleToWorkloop(void* arg) {
SWorkThrdObj* pThrd = arg;
pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
uv_loop_init(pThrd->loop);
if (0 != uv_loop_init(pThrd->loop)) {
return false;
}
// SRpcInfo* pRpc = pThrd->shandle;
uv_pipe_init(pThrd->loop, pThrd->pipe, 0);
uv_pipe_init(pThrd->loop, pThrd->pipe, 1);
uv_pipe_open(pThrd->pipe, pThrd->fd);
pThrd->pipe->data = pThrd;
......@@ -435,43 +479,79 @@ void* workerThread(void* arg) {
pThrd->workerAsync = malloc(sizeof(uv_async_t));
uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb);
pThrd->workerAsync->data = pThrd;
uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb);
return true;
}
static bool addHandleToAcceptloop(void* arg) {
// impl later
SServerObj* srv = arg;
int err = 0;
if ((err = uv_tcp_init(srv->loop, &srv->server)) != 0) {
tError("failed to init accept server: %s", uv_err_name(err));
return false;
}
struct sockaddr_in bind_addr;
uv_ip4_addr("0.0.0.0", srv->port, &bind_addr);
if ((err = uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0)) != 0) {
tError("failed to bind: %s", uv_err_name(err));
return false;
}
if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) {
tError("failed to listen: %s", uv_err_name(err));
return false;
}
return true;
}
void* workerThread(void* arg) {
SWorkThrdObj* pThrd = (SWorkThrdObj*)arg;
uv_run(pThrd->loop, UV_RUN_DEFAULT);
}
static SConn* connCreate() {
static SConn* createConn() {
SConn* pConn = (SConn*)calloc(1, sizeof(SConn));
return pConn;
}
static void connDestroy(SConn* conn) {
static void connCloseCb(uv_handle_t* handle) {
// impl later
//
}
static void destroyConn(SConn* conn, bool clear) {
if (conn == NULL) {
return;
}
if (clear) {
uv_handle_t handle = *((uv_handle_t*)conn->pTcp);
uv_close(&handle, NULL);
}
uv_timer_stop(conn->pTimer);
free(conn->pTimer);
uv_close((uv_handle_t*)conn->pTcp, NULL);
free(conn->connBuf.buf);
free(conn->pTcp);
free(conn->connBuf.buf);
free(conn->pWriter);
free(conn);
// handle
}
static void uvConnDestroy(uv_handle_t* handle) {
static void uvDestroyConn(uv_handle_t* handle) {
SConn* conn = handle->data;
connDestroy(conn);
destroyConn(conn, false);
}
static int rpcAddAuthPart(SConn* pConn, char* msg, int msgLen) {
SRpcHead* pHead = (SRpcHead*)msg;
static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) {
STransMsgHead* pHead = (STransMsgHead*)msg;
if (pConn->spi && pConn->secured == 0) {
// add auth part
pHead->spi = pConn->spi;
SRpcDigest* pDigest = (SRpcDigest*)(msg + msgLen);
STransDigestMsg* pDigest = (STransDigestMsg*)(msg + msgLen);
pDigest->timeStamp = htonl(taosGetTimestampSec());
msgLen += sizeof(SRpcDigest);
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret);
// transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret);
// transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret);
} else {
pHead->spi = 0;
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
......@@ -493,18 +573,23 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
for (int i = 0; i < srv->numOfThreads; i++) {
SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj));
srv->pThreadObj[i] = thrd;
srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t));
int fds[2];
if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) {
return NULL;
goto End;
}
uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1);
uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write
thrd->shandle = shandle;
thrd->pTransInst = shandle;
thrd->fd = fds[0];
thrd->pipe = &(srv->pipe[i][1]); // init read
if (false == addHandleToWorkloop(thrd)) {
goto End;
}
int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd));
if (err == 0) {
tDebug("sucess to create worker-thread %d", i);
......@@ -513,9 +598,10 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
// TODO: clear all other resource later
tError("failed to create worker-thread %d", i);
}
srv->pThreadObj[i] = thrd;
}
if (false == addHandleToAcceptloop(srv)) {
goto End;
}
int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv);
if (err == 0) {
tDebug("success to create accept-thread");
......@@ -524,16 +610,25 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
}
return srv;
End:
taosCloseServer(srv);
return NULL;
}
void destroyWorkThrd(SWorkThrdObj* pThrd) {
if (pThrd == NULL) {
return;
}
pthread_join(pThrd->thread, NULL);
// free(srv->pipe[i]);
free(pThrd->loop);
free(pThrd);
}
void taosCloseServer(void* arg) {
// impl later
SServerObj* srv = arg;
for (int i = 0; i < srv->numOfThreads; i++) {
SWorkThrdObj* pThrd = srv->pThreadObj[i];
pthread_join(pThrd->thread, NULL);
free(srv->pipe[i]);
free(pThrd->loop);
free(pThrd);
destroyWorkThrd(srv->pThreadObj[i]);
}
free(srv->loop);
free(srv->pipe);
......@@ -547,6 +642,7 @@ void rpcSendResponse(const SRpcMsg* pMsg) {
SWorkThrdObj* pThrd = pConn->hostThrd;
// opt later
pConn->sendMsg = *pMsg;
pthread_mutex_lock(&pThrd->connMtx);
QUEUE_PUSH(&pThrd->conn, &pConn->queue);
pthread_mutex_unlock(&pThrd->connMtx);
......
source/libs/transport/test/rclient.c
浏览文件 @ 6e358fd4
......@@ -34,8 +34,8 @@ typedef struct {
static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
SInfo *pInfo = (SInfo *)pMsg->ahandle;
// tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
// pMsg->code);
tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
pMsg->code);
if (pEpSet) pInfo->epSet = *pEpSet;
......@@ -63,6 +63,8 @@ static void *sendRequest(void *param) {
if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
// tsem_wait(&pInfo->rspSem);
tsem_wait(&pInfo->rspSem);
tDebug("recv response");
// usleep(100000000);
}
tDebug("thread:%d, it is over", pInfo->index);
......@@ -98,7 +100,7 @@ int main(int argc, char *argv[]) {
rpcInit.numOfThreads = 1;
rpcInit.cfp = processResponse;
rpcInit.sessions = 100;
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.idleTime = 100;
rpcInit.user = "michael";
rpcInit.secret = secret;
rpcInit.ckey = "key";
......
source/libs/transport/test/rserver.c
浏览文件 @ 6e358fd4
......@@ -122,7 +122,7 @@ int main(int argc, char *argv[]) {
rpcInit.numOfThreads = 1;
rpcInit.cfp = processRequestMsg;
rpcInit.sessions = 1000;
rpcInit.idleTime = tsShellActivityTimer * 1500;
rpcInit.idleTime = 2 * 1500;
rpcInit.afp = retrieveAuthInfo;
for (int i = 1; i < argc; ++i) {
......
source/util/src/compare.c
浏览文件 @ 6e358fd4
......@@ -16,13 +16,13 @@
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "types.h"
#include "compare.h"
#include "ulog.h"
#include "thash.h"
#include "regex.h"
#include "thash.h"
#include "types.h"
#include "ulog.h"
int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
......@@ -228,7 +228,7 @@ int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
* '_': Matches one character
*
*/
int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
int32_t patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
char c, c1;
int32_t i = 0;
......@@ -289,7 +289,7 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
wchar_t c, c1;
wchar_t matchOne = L'_'; // "_"
wchar_t matchAll = L'%'; // "%"
......@@ -360,11 +360,11 @@ int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
memcpy(str, varDataVal(pLeft), sz);
str[sz] = 0;
int errCode = 0;
int32_t errCode = 0;
regex_t regex;
char msgbuf[256] = {0};
int cflags = REG_EXTENDED;
int32_t cflags = REG_EXTENDED;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
......
source/util/src/tskiplist.c
浏览文件 @ 6e358fd4
......@@ -13,11 +13,13 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tskiplist.h"
#include "os.h"
#include "compare.h"
#include "ulog.h"
#include "tskiplist.h"
#include "tutil.h"
#include "ulog.h"
static int initForwardBackwardPtr(SSkipList *pSkipList);
static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur);
......@@ -52,11 +54,17 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _
pSkipList->flags = flags;
pSkipList->keyFn = fn;
pSkipList->seed = rand();
#if 0
// the function getkeycomparfunc is defined in common
if (comparFn == NULL) {
pSkipList->comparFn = getKeyComparFunc(keyType, TSDB_ORDER_ASC);
} else {
pSkipList->comparFn = comparFn;
}
#else
pSkipList->comparFn = comparFn;
#endif
if (initForwardBackwardPtr(pSkipList) < 0) {
tSkipListDestroy(pSkipList);
......
source/util/test/encodeTest.cpp
浏览文件 @ 6e358fd4
......@@ -4,6 +4,11 @@
#include "encode.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshift-count-overflow"
#pragma GCC diagnostic ignored "-Woverflow"
#pragma GCC diagnostic ignored "-Woverflow"
#define BUF_SIZE 64
td_endian_t endian_arr[2] = {TD_LITTLE_ENDIAN, TD_BIG_ENDIAN};
......@@ -354,10 +359,10 @@ TEST(td_encode_test, compound_struct_encode_test) {
SCoder encoder, decoder;
uint8_t * buf1;
int32_t buf1size;
uint8_t * buf2;
uint8_t *buf2;
int32_t buf2size;
SStructA_v1 sa1 = {.A_a = 10, .A_b = 65478, .A_c = "Hello"};
SStructA_v2 sa2 = {.A_a = 10, .A_b = 65478, .A_c = "Hello", .A_d = 67, .A_e = 13};
SStructA_v1 sa1 = {.A_a = 10, .A_b = 65478, .A_c = (char *)"Hello"};
SStructA_v2 sa2 = {.A_a = 10, .A_b = 65478, .A_c = (char *)"Hello", .A_d = 67, .A_e = 13};
SFinalReq_v1 req1 = {.pA = &sa1, .v_a = 15, .v_b = 35};
SFinalReq_v2 req2 = {.pA = &sa2, .v_a = 15, .v_b = 32, .v_c = 37};
SFinalReq_v1 dreq11, dreq21;
......@@ -431,3 +436,5 @@ TEST(td_encode_test, compound_struct_encode_test) {
GTEST_ASSERT_EQ(dreq21.v_b, req2.v_b);
tCoderClear(&decoder);
}
#pragma GCC diagnostic pop
\ No newline at end of file
tests/script/jenkins/basic.txt
浏览文件 @ 6e358fd4
......@@ -2,17 +2,17 @@
#======================b1-start===============
# ---- user
./test.sh -f sim/user/basic1.sim
./test.sh -f tsim/user/basic1.sim
# ---- db
./test.sh -f sim/db/basic1.sim
./test.sh -f sim/db/basic6.sim
./test.sh -f sim/db/error1.sim
./test.sh -f tsim/db/basic1.sim
./test.sh -f tsim/db/basic6.sim
./test.sh -f tsim/db/error1.sim
# ---- table
./test.sh -f sim/table/basic1.sim
./test.sh -f tsim/table/basic1.sim
# ---- dnode
./test.sh -f sim/dnode/basic1.sim
./test.sh -f tsim/dnode/basic1.sim
#======================b1-end===============
tests/script/sh/deploy.sh
浏览文件 @ 6e358fd4
......@@ -113,8 +113,12 @@ elif [ $NODE -eq 7 ]; then
NODE=7700
elif [ $NODE -eq 8 ]; then
NODE=7800
elif [ $NODE -eq 9 ]; then
NODE=7900
fi
HOSTNAME=localhost
echo " " >> $TAOS_CFG
echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG
echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG
......
tests/script/sh/massiveTable/setupDnodes.sh
浏览文件 @ 6e358fd4
......@@ -70,7 +70,7 @@ createNewCfgFile() {
echo "dataDir ${dataDir}" >> ${cfgFile}
echo "logDir ${logDir}" >> ${cfgFile}
echo "serverPort ${serverPort}" >> ${cfgFile}
echo "numOfLogLines 100000000" >> ${cfgFile}
echo "supportVnodes 1024" >> ${cfgFile}
#echo "asyncLog 0" >> ${cfgFile}
echo "telemetryReporting 0" >> ${cfgFile}
......
tests/script/tmp/prepare.sim
浏览文件 @ 6e358fd4
......@@ -5,7 +5,41 @@ system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
system sh/cfg.sh -n dnode3 -c walLevel -v 2
system sh/cfg.sh -n dnode4 -c walLevel -v 2
return
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/deploy.sh -n dnode5 -i 5
system sh/deploy.sh -n dnode6 -i 6
system sh/deploy.sh -n dnode7 -i 7
system sh/deploy.sh -n dnode8 -i 8
system sh/deploy.sh -n dnode9 -i 9
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
system sh/exec.sh -n dnode5 -s start
system sh/exec.sh -n dnode6 -s start
system sh/exec.sh -n dnode7 -s start
system sh/exec.sh -n dnode8 -s start
system sh/exec.sh -n dnode9 -s start
sleep 2000
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
sql create dnode $hostname port 7500
sql create dnode $hostname port 7600
sql create dnode $hostname port 7700
sql create dnode $hostname port 7800
sql create dnode $hostname port 7900
sql show dnodes;
print $data00 $data01
print $data10 $data11
print $data20 $data21
print $data30 $data31
print $data40 $data41
\ No newline at end of file
tests/test/c/create_table.c
浏览文件 @ 6e358fd4
......@@ -37,7 +37,6 @@ int32_t queryFlag = 0;
int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000
typedef struct {
int64_t tableBeginIndex;
int64_t tableEndIndex;
......@@ -89,7 +88,7 @@ void createDbAndStb() {
pRes = taos_query(con, qstr);
code = taos_errno(pRes);
if (code != 0) {
pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
pError("failed to create stable, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
exit(0);
}
taos_free_result(pRes);
......@@ -182,17 +181,13 @@ void *threadFunc(void *param) {
exit(1);
}
pError("====before thread:%d, table range: %"PRId64 " - %"PRId64 "\n",
pInfo->threadIndex,
pInfo->tableBeginIndex,
pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex,
pInfo->tableEndIndex);
pInfo->tableBeginIndex += startOffset;
pInfo->tableEndIndex += startOffset;
pError("====after thread:%d, table range: %"PRId64 " - %"PRId64 "\n",
pInfo->threadIndex,
pInfo->tableBeginIndex,
pPrint("====after thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex,
pInfo->tableEndIndex);
sprintf(qstr, "use %s", pInfo->dbName);
......@@ -221,7 +216,7 @@ void *threadFunc(void *param) {
int64_t startTs = taosGetTimestampUs();
TAOS_RES *pRes = taos_query(con, qstr);
code = taos_errno(pRes);
if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) {
if (code != 0) {
pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr);
}
taos_free_result(pRes);
......@@ -390,7 +385,7 @@ int32_t main(int32_t argc, char *argv[]) {
}
if (queryFlag) {
//selectRowsFromTable();
// selectRowsFromTable();
return 0;
}
......@@ -398,7 +393,8 @@ int32_t main(int32_t argc, char *argv[]) {
createDbAndStb();
}
pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset);
pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables,
startOffset);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
......
tests/tsim/src/simSystem.c
浏览文件 @ 6e358fd4
......@@ -42,7 +42,8 @@ char *simParseArbitratorName(char *varName) {
char *simParseHostName(char *varName) {
static char hostName[140];
sprintf(hostName, "%s", simHostName);
//sprintf(hostName, "%s", simHostName);
sprintf(hostName, "%s", "localhost");
return hostName;
}
......
tools/shell/src/backup/tnettest.c
浏览文件 @ 6e358fd4
......@@ -319,7 +319,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) {
char user[] = "nettestinternal";
char pass[] = "nettestinternal";
taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt);
taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt);
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0;
......@@ -344,7 +344,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p
SRpcMsg rspMsg;
void * pRpcConn;
char secretEncrypt[32] = {0};
char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
pRpcConn = taosNetInitRpc(secretEncrypt, spi);
if (NULL == pRpcConn) {
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或