提交 6460c712 编写于 作者: H Haojun Liao

Merge branch 'develop' into feature/query

...@@ -127,7 +127,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in ...@@ -127,7 +127,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in
hash = rpcHashConn(pCache, fqdn, port, connType); hash = rpcHashConn(pCache, fqdn, port, connType);
pNode = (SConnHash *)taosMemPoolMalloc(pCache->connHashMemPool); pNode = (SConnHash *)taosMemPoolMalloc(pCache->connHashMemPool);
strcpy(pNode->fqdn, fqdn); tstrncpy(pNode->fqdn, fqdn, sizeof(pNode->fqdn));
pNode->port = port; pNode->port = port;
pNode->connType = connType; pNode->connType = connType;
pNode->data = data; pNode->data = data;
......
...@@ -60,7 +60,7 @@ typedef struct { ...@@ -60,7 +60,7 @@ typedef struct {
void *idPool; // handle to ID pool void *idPool; // handle to ID pool
void *tmrCtrl; // handle to timer void *tmrCtrl; // handle to timer
void *hash; // handle returned by hash utility SHashObj *hash; // handle returned by hash utility
void *tcphandle;// returned handle from TCP initialization void *tcphandle;// returned handle from TCP initialization
void *udphandle;// returned handle from UDP initialization void *udphandle;// returned handle from UDP initialization
void *pCache; // connection cache void *pCache; // connection cache
...@@ -211,7 +211,7 @@ void *rpcOpen(const SRpcInit *pInit) { ...@@ -211,7 +211,7 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo));
if (pRpc == NULL) return NULL; if (pRpc == NULL) return NULL;
if(pInit->label) strcpy(pRpc->label, pInit->label); if(pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label));
pRpc->connType = pInit->connType; pRpc->connType = pInit->connType;
pRpc->idleTime = pInit->idleTime; pRpc->idleTime = pInit->idleTime;
pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads;
...@@ -228,7 +228,7 @@ void *rpcOpen(const SRpcInit *pInit) { ...@@ -228,7 +228,7 @@ void *rpcOpen(const SRpcInit *pInit) {
size_t size = sizeof(SRpcConn) * pRpc->sessions; size_t size = sizeof(SRpcConn) * pRpc->sessions;
pRpc->connList = (SRpcConn *)calloc(1, size); pRpc->connList = (SRpcConn *)calloc(1, size);
if (pRpc->connList == NULL) { if (pRpc->connList == NULL) {
tError("%s failed to allocate memory for taos connections, size:%d", pRpc->label, size); tError("%s failed to allocate memory for taos connections, size:%ld", pRpc->label, size);
rpcClose(pRpc); rpcClose(pRpc);
return NULL; return NULL;
} }
...@@ -459,7 +459,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { ...@@ -459,7 +459,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
pInfo->clientPort = pConn->peerPort; pInfo->clientPort = pConn->peerPort;
// pInfo->serverIp = pConn->destIp; // pInfo->serverIp = pConn->destIp;
strcpy(pInfo->user, pConn->user); strncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
return 0; return 0;
} }
...@@ -503,10 +503,10 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, ...@@ -503,10 +503,10 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort,
pConn = rpcAllocateClientConn(pRpc); pConn = rpcAllocateClientConn(pRpc);
if (pConn) { if (pConn) {
strcpy(pConn->peerFqdn, peerFqdn); tstrncpy(pConn->peerFqdn, peerFqdn, sizeof(pConn->peerFqdn));
pConn->peerIp = peerIp; pConn->peerIp = peerIp;
pConn->peerPort = peerPort; pConn->peerPort = peerPort;
strcpy(pConn->user, pRpc->user); tstrncpy(pConn->user, pRpc->user, sizeof(pConn->user));
pConn->connType = connType; pConn->connType = connType;
if (taosOpenConn[connType]) { if (taosOpenConn[connType]) {
...@@ -804,7 +804,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) { ...@@ -804,7 +804,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
pConn = rpcGetConnObj(pRpc, sid, pRecv); pConn = rpcGetConnObj(pRpc, sid, pRecv);
if (pConn == NULL) { if (pConn == NULL) {
tTrace("%s %p, failed to get connection obj(%s)", pRpc->label, pHead->ahandle, tstrerror(terrno)); tTrace("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno));
return NULL; return NULL;
} else { } else {
if (rpcIsReq(pHead->msgType)) { if (rpcIsReq(pHead->msgType)) {
......
...@@ -73,7 +73,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread ...@@ -73,7 +73,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1);
pServerObj->ip = ip; pServerObj->ip = ip;
pServerObj->port = port; pServerObj->port = port;
strcpy(pServerObj->label, label); tstrncpy(pServerObj->label, label, sizeof(pServerObj->label));
pServerObj->numOfThreads = numOfThreads; pServerObj->numOfThreads = numOfThreads;
pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads); pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads);
...@@ -87,7 +87,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread ...@@ -87,7 +87,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pThreadObj = pServerObj->pThreadObj; pThreadObj = pServerObj->pThreadObj;
for (int i = 0; i < numOfThreads; ++i) { for (int i = 0; i < numOfThreads; ++i) {
pThreadObj->processData = fp; pThreadObj->processData = fp;
strcpy(pThreadObj->label, label); tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
pThreadObj->shandle = shandle; pThreadObj->shandle = shandle;
code = pthread_mutex_init(&(pThreadObj->mutex), NULL); code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
...@@ -247,7 +247,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * ...@@ -247,7 +247,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *
pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj)); pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj));
memset(pThreadObj, 0, sizeof(SThreadObj)); memset(pThreadObj, 0, sizeof(SThreadObj));
strcpy(pThreadObj->label, label); tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
pThreadObj->ip = ip; pThreadObj->ip = ip;
pThreadObj->shandle = shandle; pThreadObj->shandle = shandle;
......
...@@ -72,7 +72,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads ...@@ -72,7 +72,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
pSet->port = port; pSet->port = port;
pSet->shandle = shandle; pSet->shandle = shandle;
pSet->fp = fp; pSet->fp = fp;
strcpy(pSet->label, label); tstrncpy(pSet->label, label, sizeof(pSet->label));
uint16_t ownPort; uint16_t ownPort;
for (int i = 0; i < threads; ++i) { for (int i = 0; i < threads; ++i) {
...@@ -99,7 +99,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads ...@@ -99,7 +99,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
pConn->localPort = (uint16_t)ntohs(sin.sin_port); pConn->localPort = (uint16_t)ntohs(sin.sin_port);
} }
strcpy(pConn->label, label); tstrncpy(pConn->label, label, sizeof(pConn->label));
pConn->shandle = shandle; pConn->shandle = shandle;
pConn->processData = fp; pConn->processData = fp;
pConn->index = i; pConn->index = i;
......
...@@ -76,6 +76,7 @@ int main(int argc, char *argv[]) { ...@@ -76,6 +76,7 @@ int main(int argc, char *argv[]) {
int numOfReqs = 0; int numOfReqs = 0;
int appThreads = 1; int appThreads = 1;
char serverIp[40] = "127.0.0.1"; char serverIp[40] = "127.0.0.1";
char secret[TSDB_KEY_LEN] = "mypassword";
struct timeval systemTime; struct timeval systemTime;
int64_t startTime, endTime; int64_t startTime, endTime;
pthread_attr_t thattr; pthread_attr_t thattr;
...@@ -97,7 +98,7 @@ int main(int argc, char *argv[]) { ...@@ -97,7 +98,7 @@ int main(int argc, char *argv[]) {
rpcInit.sessions = 100; rpcInit.sessions = 100;
rpcInit.idleTime = tsShellActivityTimer*1000; rpcInit.idleTime = tsShellActivityTimer*1000;
rpcInit.user = "michael"; rpcInit.user = "michael";
rpcInit.secret = "mypassword"; rpcInit.secret = secret;
rpcInit.ckey = "key"; rpcInit.ckey = "key";
rpcInit.spi = 1; rpcInit.spi = 1;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
...@@ -106,7 +107,7 @@ int main(int argc, char *argv[]) { ...@@ -106,7 +107,7 @@ int main(int argc, char *argv[]) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) { if (strcmp(argv[i], "-p")==0 && i < argc-1) {
ipSet.port[0] = atoi(argv[++i]); ipSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") ==0 && i < argc-1) { } else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
tstrncpy(ipSet.fqdn[0], argv[++i], sizeof(ipSet.fqdn)); tstrncpy(ipSet.fqdn[0], argv[++i], sizeof(ipSet.fqdn[0]));
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) { } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]); rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) { } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
......
...@@ -77,6 +77,7 @@ int main(int argc, char *argv[]) { ...@@ -77,6 +77,7 @@ int main(int argc, char *argv[]) {
int numOfReqs = 0; int numOfReqs = 0;
int appThreads = 1; int appThreads = 1;
char serverIp[40] = "127.0.0.1"; char serverIp[40] = "127.0.0.1";
char secret[TSDB_KEY_LEN] = "mypassword";
struct timeval systemTime; struct timeval systemTime;
int64_t startTime, endTime; int64_t startTime, endTime;
pthread_attr_t thattr; pthread_attr_t thattr;
...@@ -98,7 +99,7 @@ int main(int argc, char *argv[]) { ...@@ -98,7 +99,7 @@ int main(int argc, char *argv[]) {
rpcInit.sessions = 100; rpcInit.sessions = 100;
rpcInit.idleTime = tsShellActivityTimer*1000; rpcInit.idleTime = tsShellActivityTimer*1000;
rpcInit.user = "michael"; rpcInit.user = "michael";
rpcInit.secret = "mypassword"; rpcInit.secret = secret;
rpcInit.ckey = "key"; rpcInit.ckey = "key";
rpcInit.spi = 1; rpcInit.spi = 1;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
#include "tutil.h" #include "tutil.h"
int taosGetFqdn(char *fqdn) { int taosGetFqdn(char *fqdn) {
int code = 0;
char hostname[1024]; char hostname[1024];
hostname[1023] = '\0'; hostname[1023] = '\0';
gethostname(hostname, 1023); gethostname(hostname, 1023);
...@@ -27,13 +28,15 @@ int taosGetFqdn(char *fqdn) { ...@@ -27,13 +28,15 @@ int taosGetFqdn(char *fqdn) {
h = gethostbyname(hostname); h = gethostbyname(hostname);
if (h != NULL) { if (h != NULL) {
strcpy(fqdn, h->h_name); strcpy(fqdn, h->h_name);
return 0;
} else { } else {
uError("failed to get host name"); uError("failed to get host name(%s)", strerror(errno));
return -1; code = -1;
} }
free(h); // to do: free the resources
// free(h);
return code;
} }
uint32_t taosGetIpFromFqdn(const char *fqdn) { uint32_t taosGetIpFromFqdn(const char *fqdn) {
...@@ -47,7 +50,7 @@ uint32_t ip2uint(const char *const ip_addr) { ...@@ -47,7 +50,7 @@ uint32_t ip2uint(const char *const ip_addr) {
char ip_addr_cpy[20]; char ip_addr_cpy[20];
char ip[5]; char ip[5];
strcpy(ip_addr_cpy, ip_addr); tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy));
char *s_start, *s_end; char *s_start, *s_end;
s_start = ip_addr_cpy; s_start = ip_addr_cpy;
...@@ -206,7 +209,7 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) { ...@@ -206,7 +209,7 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
int reuse, nocheck; int reuse, nocheck;
int bufSize = 8192000; int bufSize = 8192000;
uTrace("open udp socket:%s:%hu", ip, port); uTrace("open udp socket:0x%x:%hu", ip, port);
memset((char *)&localAddr, 0, sizeof(localAddr)); memset((char *)&localAddr, 0, sizeof(localAddr));
localAddr.sin_family = AF_INET; localAddr.sin_family = AF_INET;
...@@ -257,7 +260,7 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) { ...@@ -257,7 +260,7 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
/* bind socket to local address */ /* bind socket to local address */
if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) {
uError("failed to bind udp socket: %d (%s), %s:%hu", errno, strerror(errno), ip, port); uError("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port);
taosCloseSocket(sockFd); taosCloseSocket(sockFd);
return -1; return -1;
} }
...@@ -363,7 +366,7 @@ int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { ...@@ -363,7 +366,7 @@ int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
int sockFd; int sockFd;
int reuse; int reuse;
uTrace("open tcp server socket:%s:%hu", ip, port); uTrace("open tcp server socket:0x%x:%hu", ip, port);
bzero((char *)&serverAdd, sizeof(serverAdd)); bzero((char *)&serverAdd, sizeof(serverAdd));
serverAdd.sin_family = AF_INET; serverAdd.sin_family = AF_INET;
......
...@@ -68,11 +68,19 @@ void *walOpen(const char *path, const SWalCfg *pCfg) { ...@@ -68,11 +68,19 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
pWal->num = 0; pWal->num = 0;
pWal->level = pCfg->walLevel; pWal->level = pCfg->walLevel;
pWal->keep = pCfg->keep; pWal->keep = pCfg->keep;
strcpy(pWal->path, path); tstrncpy(pWal->path, path, sizeof(pWal->path));
pthread_mutex_init(&pWal->mutex, NULL); pthread_mutex_init(&pWal->mutex, NULL);
if (access(path, F_OK) != 0) mkdir(path, 0755); if (access(path, F_OK) != 0) {
if (mkdir(path, 0755) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
pthread_mutex_destroy(&pWal->mutex);
free(pWal);
pWal = NULL;
}
}
if (pCfg->keep == 1) return pWal; if (pCfg->keep == 1) return pWal;
if (walHandleExistingFiles(path) == 0) if (walHandleExistingFiles(path) == 0)
...@@ -80,7 +88,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) { ...@@ -80,7 +88,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
if (pWal->fd <0) { if (pWal->fd <0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("wal:%s, failed to open", path); wError("wal:%s, failed to open(%s)", path, strerror(errno));
pthread_mutex_destroy(&pWal->mutex); pthread_mutex_destroy(&pWal->mutex);
free(pWal); free(pWal);
pWal = NULL; pWal = NULL;
...@@ -119,7 +127,8 @@ void walClose(void *handle) { ...@@ -119,7 +127,8 @@ void walClose(void *handle) {
int walRenew(void *handle) { int walRenew(void *handle) {
if (handle == NULL) return 0; if (handle == NULL) return 0;
SWal *pWal = handle; SWal *pWal = handle;
int code = 0;
terrno = 0;
pthread_mutex_lock(&pWal->mutex); pthread_mutex_lock(&pWal->mutex);
...@@ -135,8 +144,8 @@ int walRenew(void *handle) { ...@@ -135,8 +144,8 @@ int walRenew(void *handle) {
pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (pWal->fd < 0) { if (pWal->fd < 0) {
wError("wal:%d, failed to open(%s)", pWal->name, strerror(errno)); wError("wal:%s, failed to open(%s)", pWal->name, strerror(errno));
code = -1; terrno = TAOS_SYSTEM_ERROR(errno);
} else { } else {
wTrace("wal:%s, it is created", pWal->name); wTrace("wal:%s, it is created", pWal->name);
...@@ -156,14 +165,15 @@ int walRenew(void *handle) { ...@@ -156,14 +165,15 @@ int walRenew(void *handle) {
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
return code; return terrno;
} }
int walWrite(void *handle, SWalHead *pHead) { int walWrite(void *handle, SWalHead *pHead) {
SWal *pWal = handle; SWal *pWal = handle;
int code = 0;
if (pWal == NULL) return -1; if (pWal == NULL) return -1;
terrno = 0;
// no wal // no wal
if (pWal->level == TAOS_WAL_NOLOG) return 0; if (pWal->level == TAOS_WAL_NOLOG) return 0;
if (pHead->version <= pWal->version) return 0; if (pHead->version <= pWal->version) return 0;
...@@ -174,12 +184,12 @@ int walWrite(void *handle, SWalHead *pHead) { ...@@ -174,12 +184,12 @@ int walWrite(void *handle, SWalHead *pHead) {
if(write(pWal->fd, pHead, contLen) != contLen) { if(write(pWal->fd, pHead, contLen) != contLen) {
wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno)); wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
code = -1; terrno = TAOS_SYSTEM_ERROR(errno);
} else { } else {
pWal->version = pHead->version; pWal->version = pHead->version;
} }
return code; return terrno;
} }
void walFsync(void *handle) { void walFsync(void *handle) {
...@@ -196,11 +206,11 @@ void walFsync(void *handle) { ...@@ -196,11 +206,11 @@ void walFsync(void *handle) {
int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) { int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) {
SWal *pWal = handle; SWal *pWal = handle;
int code = 0;
struct dirent *ent; struct dirent *ent;
int count = 0; int count = 0;
uint32_t maxId = 0, minId = -1, index =0; uint32_t maxId = 0, minId = -1, index =0;
terrno = 0;
int plen = strlen(walPrefix); int plen = strlen(walPrefix);
char opath[TSDB_FILENAME_LEN+5]; char opath[TSDB_FILENAME_LEN+5];
...@@ -224,30 +234,30 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) ...@@ -224,30 +234,30 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
closedir(dir); closedir(dir);
if (count == 0) { if (count == 0) {
if (pWal->keep) code = walRenew(pWal); if (pWal->keep) terrno = walRenew(pWal);
return code; return terrno;
} }
if ( count != (maxId-minId+1) ) { if ( count != (maxId-minId+1) ) {
wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId); wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId);
code = -1; terrno = TAOS_SYSTEM_ERROR(TSDB_CODE_APP_ERROR);
} else { } else {
wTrace("wal:%s, %d files will be restored", opath, count); wTrace("wal:%s, %d files will be restored", opath, count);
for (index = minId; index<=maxId; ++index) { for (index = minId; index<=maxId; ++index) {
sprintf(pWal->name, "%s/%s%d", opath, walPrefix, index); sprintf(pWal->name, "%s/%s%d", opath, walPrefix, index);
code = walRestoreWalFile(pWal, pVnode, writeFp); terrno = walRestoreWalFile(pWal, pVnode, writeFp);
if (code < 0) break; if (terrno < 0) break;
} }
} }
if (code == 0) { if (terrno == 0) {
if (pWal->keep == 0) { if (pWal->keep == 0) {
code = walRemoveWalFiles(opath); terrno = walRemoveWalFiles(opath);
if (code == 0) { if (terrno == 0) {
if (remove(opath) < 0) { if (remove(opath) < 0) {
wError("wal:%s, failed to remove directory(%s)", opath, strerror(errno)); wError("wal:%s, failed to remove directory(%s)", opath, strerror(errno));
code = -1; terrno = TAOS_SYSTEM_ERROR(errno);
} }
} }
} else { } else {
...@@ -258,12 +268,12 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) ...@@ -258,12 +268,12 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
if (pWal->fd < 0) { if (pWal->fd < 0) {
wError("wal:%s, failed to open file(%s)", pWal->name, strerror(errno)); wError("wal:%s, failed to open file(%s)", pWal->name, strerror(errno));
code = -1; terrno = TAOS_SYSTEM_ERROR(errno);
} }
} }
} }
return code; return terrno;
} }
int walGetWalFile(void *handle, char *name, uint32_t *index) { int walGetWalFile(void *handle, char *name, uint32_t *index) {
...@@ -292,40 +302,47 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) { ...@@ -292,40 +302,47 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) {
} }
static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
int code = 0;
char *name = pWal->name; char *name = pWal->name;
terrno = 0;
char *buffer = malloc(1024000); // size for one record char *buffer = malloc(1024000); // size for one record
if (buffer == NULL) return -1; if (buffer == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
SWalHead *pHead = (SWalHead *)buffer; SWalHead *pHead = (SWalHead *)buffer;
int fd = open(name, O_RDONLY); int fd = open(name, O_RDONLY);
if (fd < 0) { if (fd < 0) {
wError("wal:%s, failed to open for restore(%s)", name, strerror(errno)); wError("wal:%s, failed to open for restore(%s)", name, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
free(buffer); free(buffer);
return -1; return terrno;
} }
wTrace("wal:%s, start to restore", name); wTrace("wal:%s, start to restore", name);
while (1) { while (1) {
int ret = read(fd, pHead, sizeof(SWalHead)); int ret = read(fd, pHead, sizeof(SWalHead));
if ( ret == 0) { code = 0; break;} if ( ret == 0) break;
if (ret != sizeof(SWalHead)) { if (ret != sizeof(SWalHead)) {
wWarn("wal:%s, failed to read head, skip, ret:%d(%s)", name, ret, strerror(errno)); wWarn("wal:%s, failed to read head, skip, ret:%d(%s)", name, ret, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
break; break;
} }
if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
wWarn("wal:%s, cksum is messed up, skip the rest of file", name); wWarn("wal:%s, cksum is messed up, skip the rest of file", name);
terrno = TAOS_SYSTEM_ERROR(errno);
break; break;
} }
ret = read(fd, pHead->cont, pHead->len); ret = read(fd, pHead->cont, pHead->len);
if ( ret != pHead->len) { if ( ret != pHead->len) {
wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret); wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret);
terrno = TAOS_SYSTEM_ERROR(errno);
break; break;
} }
...@@ -336,11 +353,10 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { ...@@ -336,11 +353,10 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
close(fd); close(fd);
free(buffer); free(buffer);
return code; return terrno;
} }
int walHandleExistingFiles(const char *path) { int walHandleExistingFiles(const char *path) {
int code = 0;
char oname[TSDB_FILENAME_LEN * 3]; char oname[TSDB_FILENAME_LEN * 3];
char nname[TSDB_FILENAME_LEN * 3]; char nname[TSDB_FILENAME_LEN * 3];
char opath[TSDB_FILENAME_LEN]; char opath[TSDB_FILENAME_LEN];
...@@ -350,6 +366,7 @@ int walHandleExistingFiles(const char *path) { ...@@ -350,6 +366,7 @@ int walHandleExistingFiles(const char *path) {
struct dirent *ent; struct dirent *ent;
DIR *dir = opendir(path); DIR *dir = opendir(path);
int plen = strlen(walPrefix); int plen = strlen(walPrefix);
terrno = 0;
if (access(opath, F_OK) == 0) { if (access(opath, F_OK) == 0) {
// old directory is there, it means restore process is not finished // old directory is there, it means restore process is not finished
...@@ -360,13 +377,19 @@ int walHandleExistingFiles(const char *path) { ...@@ -360,13 +377,19 @@ int walHandleExistingFiles(const char *path) {
int count = 0; int count = 0;
while ((ent = readdir(dir))!= NULL) { while ((ent = readdir(dir))!= NULL) {
if ( strncmp(ent->d_name, walPrefix, plen) == 0) { if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
if (access(opath, F_OK) != 0) mkdir(opath, 0755);
sprintf(oname, "%s/%s", path, ent->d_name); sprintf(oname, "%s/%s", path, ent->d_name);
sprintf(nname, "%s/old/%s", path, ent->d_name); sprintf(nname, "%s/old/%s", path, ent->d_name);
if (access(opath, F_OK) != 0) {
if (mkdir(opath, 0755) != 0) {
wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
break;
}
}
if (rename(oname, nname) < 0) { if (rename(oname, nname) < 0) {
wError("wal:%s, failed to move to new:%s", oname, nname); wError("wal:%s, failed to move to new:%s", oname, nname);
code = -1; terrno = TAOS_SYSTEM_ERROR(errno);
break; break;
} }
...@@ -378,14 +401,14 @@ int walHandleExistingFiles(const char *path) { ...@@ -378,14 +401,14 @@ int walHandleExistingFiles(const char *path) {
} }
closedir(dir); closedir(dir);
return code; return terrno;
} }
static int walRemoveWalFiles(const char *path) { static int walRemoveWalFiles(const char *path) {
int plen = strlen(walPrefix); int plen = strlen(walPrefix);
char name[TSDB_FILENAME_LEN * 3]; char name[TSDB_FILENAME_LEN * 3];
int code = 0;
terrno = 0;
if (access(path, F_OK) != 0) return 0; if (access(path, F_OK) != 0) return 0;
struct dirent *ent; struct dirent *ent;
...@@ -396,13 +419,13 @@ static int walRemoveWalFiles(const char *path) { ...@@ -396,13 +419,13 @@ static int walRemoveWalFiles(const char *path) {
sprintf(name, "%s/%s", path, ent->d_name); sprintf(name, "%s/%s", path, ent->d_name);
if (remove(name) <0) { if (remove(name) <0) {
wError("wal:%s, failed to remove(%s)", name, strerror(errno)); wError("wal:%s, failed to remove(%s)", name, strerror(errno));
code = -1; break; terrno = TAOS_SYSTEM_ERROR(errno);
} }
} }
} }
closedir(dir); closedir(dir);
return code; return terrno;
} }
...@@ -2,11 +2,11 @@ ...@@ -2,11 +2,11 @@
# step 1: start dnode1 # step 1: start dnode1
# step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode) # step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
# step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841) # step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
# step 4: insert old data(now-15d) and new data(now+15d), control data rows in order to save in cache, not falling disc # step 4: insert old data(now-20d) and new data(now-40d), control data rows in order to save in cache, not falling disc
# step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2 # step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
# step 6: insert two data rows: now-16d, now+16d # step 6: insert two data rows: now-21d, now-41d
# step 7: restart dnode2, waiting sync end # step 7: restart dnode2, waiting sync end
# expect: in dnode2, the files 1840 and 1842 will be removed # expect: in dnode2, the files 1837 and 1839 will be removed
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
...@@ -14,10 +14,10 @@ system sh/deploy.sh -n dnode2 -i 2 ...@@ -14,10 +14,10 @@ system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2
...@@ -39,11 +39,6 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 ...@@ -39,11 +39,6 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
...@@ -64,16 +59,16 @@ sql connect ...@@ -64,16 +59,16 @@ sql connect
print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data
system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode2 -s start
#system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode3 -s start
sql create dnode $hostname2 sql create dnode $hostname2
#sql create dnode $hostname3 sql create dnode $hostname3
sleep 3000 sleep 3000
$totalTableNum = 1 $totalTableNum = 1
$sleepTimer = 3000 $sleepTimer = 3000
$db = db $db = db
sql create database $db replica 1 cache 1 sql create database $db replica 2 cache 1
sql use $db sql use $db
# create table , insert data # create table , insert data
...@@ -82,7 +77,7 @@ sql create table $stb (ts timestamp, c1 double) tags(t1 int) ...@@ -82,7 +77,7 @@ sql create table $stb (ts timestamp, c1 double) tags(t1 int)
$rowNum = 130000 $rowNum = 130000
$tblNum = $totalTableNum $tblNum = $totalTableNum
$totalRows = 0 $totalRows = 0
#$tsStart = 1420041600000 $tsStart = 1577808000000 # 2020-01-01 00:00:00.000
# insert over 2M data in order to falling disc, generate one file # insert over 2M data in order to falling disc, generate one file
$i = 0 $i = 0
...@@ -102,19 +97,24 @@ while $i < $tblNum ...@@ -102,19 +97,24 @@ while $i < $tblNum
endw endw
sql select count(*) from $stb sql select count(*) from $stb
sleep 1000 print rows:$rows data00:$data00
print data00 $data00 if $rows != 1 then
if $data00 != $totalRows then return -1
return -1 endi
if $data00 == 0 then
return -1
endi endi
$totalRows = $data00
print ============== step3: insert old data(now-15d) and new data(now+15d), control data rows in order to save in cache, not falling disc print ============== step3: insert old data(now-15d) and new data(now+15d), control data rows in order to save in cache, not falling disc
sql insert into $tb values ( now - 15d , -15 ) sql insert into $tb values ( now - 20d , -20 )
sql insert into $tb values ( now + 15d , 15 ) sql insert into $tb values ( now - 40d , -40 )
$totalRows = $totalRows + 2 $totalRows = $totalRows + 2
print ============== step4: stop dnode2, so date rows falling disc, generate two new files in dnode2 print ============== step4: stop dnode2, so date rows falling disc, generate two new files in dnode2
system sh/exec.sh -n dnode2 -s stop system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep $sleepTimer sleep $sleepTimer
wait_dnode2_offline: wait_dnode2_offline:
...@@ -153,48 +153,14 @@ if $data00 != $totalRows then ...@@ -153,48 +153,14 @@ if $data00 != $totalRows then
endi endi
print ============== step5: insert two data rows: now-16d, now+16d, print ============== step5: insert two data rows: now-16d, now+16d,
sql insert into $tb values ( now - 16d , -16 ) sql insert into $tb values ( now - 21d , -21 )
sql insert into $tb values ( now + 16d , 16 ) sql insert into $tb values ( now - 41d , -41 )
$totalRows = $totalRows + 2 $totalRows = $totalRows + 2
return 1
print ============== step5: restart dnode2, waiting sync end
system sh/exec.sh -n dnode2 -s start
sleep 3000
wait_dnode2_ready:
sql show dnodes
if $rows != 3 then
sleep 2000
goto wait_dnode2_ready
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
#$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
#$dnode4Status = $data4_4
#$dnode5Status = $data4_5
if $dnode2Status != ready then
sleep 2000
goto wait_dnode2_ready
endi
sleep $sleepTimer
# check using select
sql select count(*) from $stb sql select count(*) from $stb
print data00 $data00 print data00 $data00
if $data00 != $totalRows then if $data00 != $totalRows then
return -1 return -1
endi endi
print ============== step6: in dnode2, the files 1840 and 1842 will be removed print ============== step6: please check there should be 3 file in sim/dnode2/data/vnode/vnode2/tsdb/data/, and 1 file sim/dnode3/data/vnode/vnode2/tsdb/data/
# how check in script ???
\ No newline at end of file
# Test case describe: dnode1 is only mnode, dnode2/dnode3 are only vnode
# step 1: start dnode1
# step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
# step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
# step 4: insert old data(now-20d) and new data(now-40d), control data rows in order to save in cache, not falling disc
# step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
# step 6: insert two data rows: now-21d, now-41d
# step 7: restart dnode2, waiting sync end
# expect: in dnode2, the files 1837 and 1839 will be removed
sql connect
sleep 3000
print ============== step7: restart dnode2, waiting sync end
system sh/exec.sh -n dnode2 -s start
sleep 3000
wait_dnode2_ready:
sql show dnodes
if $rows != 3 then
sleep 2000
goto wait_dnode2_ready
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
#$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
#$dnode4Status = $data4_4
#$dnode5Status = $data4_5
if $dnode2Status != ready then
sleep 2000
goto wait_dnode2_ready
endi
sleep 1000
# check using select
$db = db
$stb = stb
sql use $db
sql select count(*) from $stb
print data00 $data00, should equal to dn2_mn1_cache_file_sync.sim output
#if $data00 != $totalRows then
# return -1
#endi
print ============== step8: please check there should be 1 file in sim/dnode2/data/vnode/vnode2/tsdb/data/, and 1 file sim/dnode3/data/vnode/vnode2/tsdb/data/
...@@ -23,28 +23,39 @@ system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 ...@@ -23,28 +23,39 @@ system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 200 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 200 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 200
#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200 #system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200
#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200 #system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200 #system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200
system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode1 -c offlineThreshold -v 20 system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode2 -c offlineThreshold -v 20 system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode3 -c offlineThreshold -v 20 system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode4 -c offlineThreshold -v 20 system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode5 -c offlineThreshold -v 20 system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1
system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1
...@@ -63,9 +74,9 @@ sql connect ...@@ -63,9 +74,9 @@ sql connect
print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data
system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode3 -s start
sleep 1000
sql create dnode $hostname2 sql create dnode $hostname2
sql create dnode $hostname3 sql create dnode $hostname3
sleep 3000
$rowNum = 100 $rowNum = 100
$tblNum = 16 $tblNum = 16
...@@ -151,7 +162,7 @@ endi ...@@ -151,7 +162,7 @@ endi
# return -1 # return -1
#endi #endi
sleep 30000 sleep 15000
wait_drop: wait_drop:
sql show dnodes sql show dnodes
...@@ -174,10 +185,69 @@ endi ...@@ -174,10 +185,69 @@ endi
if $dnode2Status != ready then if $dnode2Status != ready then
return -1 return -1
endi endi
if $dnode3Status != null then
return -1
endi
if $dnode4Status != ready then if $dnode4Status != ready then
return -1 return -1
endi endi
print ============== step4-1: restart dnode3, adn add into cluster
system rm -rf ../../sim/dnode3
sleep 3000
system sh/deploy.sh -n dnode3 -i 3
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c walLevel -v 1
system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
system sh/exec.sh -n dnode3 -s start
sql create dnode $hostname3
sleep 3000
wait_dnode3_ready:
sql show dnodes
print rows: $rows
if $rows != 4 then
sleep 3000
goto wait_dnode3_ready
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
$dnode4Status = $data4_4
$dnode5Status = $data4_5
if $dnode1Status != ready then
return -1
endi
if $dnode2Status != ready then
return -1
endi
if $dnode3Status != null then
return -1
endi
if $dnode4Status != ready then
return -1
endi
if $dnode5Status != ready then
return -1
endi
sql select count(*) from $stb
print data00 $data00
if $data00 != $totalRows then
return -1
endi
print ============== step5: start dnode5 and add into cluster , drop database print ============== step5: start dnode5 and add into cluster , drop database
sql drop database $db sql drop database $db
...@@ -187,7 +257,7 @@ sql create dnode $hostname5 ...@@ -187,7 +257,7 @@ sql create dnode $hostname5
sleep 3000 sleep 3000
wait_dnode5: wait_dnode5:
sql show dnodes sql show dnodes
if $rows != 4 then if $rows != 5 then
sleep 3000 sleep 3000
goto wait_dnode5 goto wait_dnode5
endi endi
...@@ -196,10 +266,12 @@ print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 ...@@ -196,10 +266,12 @@ print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
$dnode1Status = $data4_1 $dnode1Status = $data4_1
$dnode2Status = $data4_2 $dnode2Status = $data4_2
$dnode4Status = $data4_4 $dnode4Status = $data4_4
$dnode5Status = $data4_5 $dnode5Status = $data4_5
$dnode6Status = $data4_6
if $dnode1Status != ready then if $dnode1Status != ready then
return -1 return -1
...@@ -207,17 +279,22 @@ endi ...@@ -207,17 +279,22 @@ endi
if $dnode2Status != ready then if $dnode2Status != ready then
return -1 return -1
endi endi
if $dnode3Status != null then
return -1
endi
if $dnode4Status != ready then if $dnode4Status != ready then
return -1 return -1
endi endi
if $dnode5Status != ready then if $dnode5Status != ready then
return -1 return -1
endi endi
if $dnode6Status != ready then
return -1
endi
print ============== step6: create database and table until not free vnodes print ============== step6: create database and table until not free vnodes
$rowNum = 100 $rowNum = 100
$tblNum = 24 $tblNum = 32
$totalRows = 0 $totalRows = 0
$tsStart = 1420041600000 $tsStart = 1420041600000
...@@ -259,8 +336,9 @@ if $data00 != $totalRows then ...@@ -259,8 +336,9 @@ if $data00 != $totalRows then
return -1 return -1
endi endi
print ============== step7: drop dnode3, and system should prompt cannot drop dnodes print ============== step7: drop dnode $hostname5, system should prompt "DB error: no enough dnodes"
sql_error drop dnode $hostname3 sql_error drop dnode $hostname5
print ============== step8: add one new table, and system should prompt 'need more dnode' print error: $error
print ============== step8: create table tb_more using $stb tags( 1000 ), system should prompt 'DB error: no enough dnodes'
sql_error create table tb_more using $stb tags( 1000 ) sql_error create table tb_more using $stb tags( 1000 )
print error: $error
...@@ -89,11 +89,8 @@ if $data00 != $totalRows then ...@@ -89,11 +89,8 @@ if $data00 != $totalRows then
return -1 return -1
endi endi
print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows
system sh/exec.sh -n dnode2 -s stop -x SIGINT
print ============== step2-1: start dnode2 for falling disc, then restart dnode2, and check rows
system sh/exec.sh -n dnode2 -s stop
sleep $sleepTimer sleep $sleepTimer
wait_dnode2_offline_0: wait_dnode2_offline_0:
sql show dnodes sql show dnodes
...@@ -151,10 +148,6 @@ if $data00 != $totalRows then ...@@ -151,10 +148,6 @@ if $data00 != $totalRows then
endi endi
print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync
system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode3 -s start
sql create dnode $hostname3 sql create dnode $hostname3
......
# Test case describe: dnode1 is only mnode, dnode2/dnode3 are only vnode
# step 1: start dnode1
# step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
# step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
# step 4: insert old data(now-15d) and new data(now+15d), control data rows in order to save in cache, not falling disc
# step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
# step 6: insert two data rows: now-16d, now+16d
# step 7: restart dnode2, waiting sync end
# expect: in dnode2, the files 1840 and 1842 will be removed
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
system sh/cfg.sh -n dnode3 -c walLevel -v 2
system sh/cfg.sh -n dnode4 -c walLevel -v 2
system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start
print ============== step1: start dnode1, only deploy mnode
system sh/exec.sh -n dnode1 -s start
sleep 3000
sql connect
print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
sql create dnode $hostname2
sql create dnode $hostname3
sleep 3000
$totalTableNum = 1
$sleepTimer = 3000
$db = db
sql create database $db replica 2 cache 1
sql use $db
# create table , insert data
$stb = stb
sql create table $stb (ts timestamp, c1 double) tags(t1 int)
$rowNum = 130000
$tblNum = $totalTableNum
$totalRows = 0
$tsStart = 1577808000000 # 2020-01-01 00:00:00.000
# insert over 2M data in order to falling disc, generate one file
$i = 0
while $i < $tblNum
$tb = tb . $i
sql create table $tb using $stb tags( $i )
$x = 0
while $x < $rowNum
# $ts = $tsStart + $x
sql insert into $tb values ( now + 0s , $x ) ( now + 1s , $x ) ( now + 2s , $x ) ( now + 3s , $x ) ( now + 4s , $x ) ( now + 5s , $x ) ( now + 6s , $x ) ( now + 7s , $x ) ( now + 8s , $x ) ( now + 9s , $x )
$x = $x + 10
endw
$totalRows = $totalRows + $x
print info: inserted $x rows into $tb and totalRows: $totalRows
$i = $i + 1
endw
sql select count(*) from $stb
print rows:$rows data00:$data00
if $rows != 1 then
return -1
endi
if $data00 == 0 then
return -1
endi
$totalRows = $data00
print ============== step3: insert old data(now-15d) and new data(now+15d), control data rows in order to save in cache, not falling disc
sql insert into $tb values ( now - 20d , -20 )
sql insert into $tb values ( now - 40d , -40 )
$totalRows = $totalRows + 2
print ============== step4: stop dnode2, so date rows falling disc, generate two new files in dnode2
system sh/exec.sh -n dnode2 -s stop -x SIGINT
sleep $sleepTimer
wait_dnode2_offline:
sql show dnodes
if $rows != 3 then
sleep 2000
goto wait_dnode2_offline
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
#$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
#$dnode4Status = $data4_4
#$dnode5Status = $data4_5
if $dnode2Status != offline then
sleep 2000
goto wait_dnode2_offline
endi
if $dnode3Status != ready then
sleep 2000
goto wait_dnode2_offline
endi
sleep $sleepTimer # waitting for move master vnode of dnode2 to dnode3
# check using select
sql select count(*) from $stb
print data00 $data00
if $data00 != $totalRows then
return -1
endi
print ============== step5: insert two data rows: now-16d, now+16d,
sql insert into $tb values ( now - 21d , -21 )
sql insert into $tb values ( now - 41d , -41 )
$totalRows = $totalRows + 2
print ============== step5: restart dnode2, waiting sync end
system sh/exec.sh -n dnode2 -s start
sleep 3000
wait_dnode2_ready:
sql show dnodes
if $rows != 3 then
sleep 2000
goto wait_dnode2_ready
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
#$dnode1Status = $data4_1
$dnode2Status = $data4_2
$dnode3Status = $data4_3
#$dnode4Status = $data4_4
#$dnode5Status = $data4_5
if $dnode2Status != ready then
sleep 2000
goto wait_dnode2_ready
endi
sleep $sleepTimer
# check using select
sql select count(*) from $stb
print data00 $data00
if $data00 != $totalRows then
return -1
endi
...@@ -2,18 +2,42 @@ system sh/stop_dnodes.sh ...@@ -2,18 +2,42 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode2 -c walLevel -v 2
system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode3 -c walLevel -v 2
system sh/cfg.sh -n dnode4 -c walLevel -v 2
system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 8
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 8
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 8
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c alternativeRole -v 0
system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
system sh/cfg.sh -n dnode3 -c alternativeRole -v 0
system sh/cfg.sh -n dnode4 -c alternativeRole -v 0
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
...@@ -22,21 +46,23 @@ system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator ...@@ -22,21 +46,23 @@ system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
print ============== step0: start tarbitrator print ============== step0: start tarbitrator
system sh/exec_tarbitrator.sh -s start system sh/exec_tarbitrator.sh -s start
print ============== step1: replica is 1, and start 1 dnode, then create tables and insert data
print ============== step1: replica is 1, and start 1 dnode
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 3000 sleep 3000
sql connect sql connect
$db = replica_db1 $totalTableNum = 12
sql create database $db replica 1 maxTables 4
$db = db
sql create database $db replica 1 maxTables $totalTableNum
sql use $db sql use $db
# create table , insert data # create table , insert data
$stb = repl_stb $stb = stb
sql create table $stb (ts timestamp, c1 int) tags(t1 int) sql create table $stb (ts timestamp, c1 int) tags(t1 int)
$rowNum = 10 $rowNum = 10
$tblNum = 12 $tblNum = $totalTableNum
$totalRows = $rowNum * $tblNum
$ts0 = 1420041600000 $ts0 = 1420041600000
$ts = $ts0 $ts = $ts0
...@@ -55,46 +81,136 @@ while $i < $tblNum ...@@ -55,46 +81,136 @@ while $i < $tblNum
$x = $x + 1 $x = $x + 1
endw endw
$i = $i + 1 $i = $i + 1
print $tb inserted rows: $x
endw endw
sql select count(*) from $stb
print data00 $data00
if $data00 != $totalRows then
return -1
endi
print ============== step2: add 1 new dnode, expect balanced print ============== step2: add 1 new dnode, expect balanced
system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode2 -s start
sql create dnode $hostname2 sql create dnode $hostname2
sleep 3000
# expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2 # expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2
$x = 0 wait_dnode2_ready:
show2:
$x = $x + 1
sleep 2000
if $x == 10 then
return -1
endi
sql show dnodes sql show dnodes
print dnode1 openVnodes $data2_1 if $rows != 2 then
print dnode2 openVnodes $data2_2 sleep 2000
if $data2_1 != 2 then goto wait_dnode2_ready
goto show2 endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
$dnode1Status = $data4_1
$dnode2Status = $data4_2
#$dnode3Status = $data4_3
#$dnode4Status = $data4_4
#$dnode5Status = $data4_5
if $dnode1Status != ready then
sleep 2000
goto wait_dnode2_ready
endi endi
if $data2_2 != 1 then if $dnode2Status != ready then
goto show2 sleep 2000
goto wait_dnode2_ready
endi endi
print ============== step4: stop dnode1, and wait dnode2 master print ============== step3: stop dnode1/dnode2, modify cfg mpeers to 2, and restart dnode1/dnode2
system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode1 -s stop
system sh/exec.sh -n dnode2 -s stop
sleep 3000
$x = 0 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
loop_wait: system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
$x = $x + 1
sleep 2000 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
if $x == 10 then system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
print ERROR: after dnode1 stop, dnode2 didn't become a master!
return -1 system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
endi system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
sleep 5000
print ============= step4: wait dnode ready
wait_dnode_ready:
sql show dnodes
if $rows != 2 then
sleep 2000
goto wait_dnode_ready
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
#print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
$dnode1Status = $data4_1
$dnode2Status = $data4_2
#$dnode3Status = $data4_3
#$dnode4Status = $data4_4
#$dnode5Status = $data4_5
if $dnode1Status != ready then
sleep 2000
goto wait_dnode_ready
endi
if $dnode2Status != ready then
sleep 2000
goto wait_dnode_ready
endi
sql select count(*) from $stb
print data00 $data00
if $data00 != $totalRows then
return -1
endi
print ============== step5: stop dnode1
system sh/exec.sh -n dnode1 -s stop
sleep 3000
wait_dnode2_master:
sql show mnodes sql show mnodes
$dnodeRole = $data2_1 if $rows != 2 then
print dnodeRole ==> $dnodeRole sleep 2000
goto wait_dnode2_master
endi
print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1
print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2
#print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3
#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4
#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5
#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6
$dnode1Status = $data4_1
$dnode2Status = $data4_2
#$dnode3Status = $data4_3
#$dnode4Status = $data4_4
#$dnode5Status = $data4_5
if $data2_1 != offline then
sleep 2000
goto wait_dnode2_master
endi
if $data2_2 != master then
sleep 2000
goto wait_dnode2_master
endi
if $dnodeRole != master then sql select count(*) from $stb
goto loop_wait print data00 $data00
if $data00 != $totalRows then
return -1
endi endi
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册