diff --git a/Jenkinsfile b/Jenkinsfile index 3968451d875229fa4db9195c014081be10297dc1..8bf7e435fd5e96aa49e7f2e07af0c4b2decc365e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,6 +86,13 @@ pipeline { ./crash_gen.sh -a -p -t 4 -s 2000 ''' } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4 + ./handle_crash_gen_val_log.sh + ''' + } sh ''' date cd ${WKC}/tests @@ -131,14 +138,33 @@ pipeline { sh''' cd ${WORKSPACE} git checkout develop - cd tests/gotest - bash batchtest.sh - cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single >/dev/null - java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 - cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker - python3 PythonChecker.py ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/gotest + bash batchtest.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker + python3 PythonChecker.py + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single >/dev/null + java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# + dotnet run + ''' + } + } } @@ -146,5 +172,82 @@ pipeline { } } - + post { + success { + emailext ( + subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 变更概要：${CHANGES}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 变更概要：${CHANGES}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + } } \ No newline at end of file diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index c996bb2a76ff505fca4e09cd8763b324e5c4cb8d..64ba0dbf3219948d420b9da8420b1c8f93e47fb0 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -427,8 +427,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } else { assert(code == TSDB_CODE_SUCCESS); } - - assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL); + // param already freed by other routine and pSql in tscCache when ctrl + c + if (atomic_load_ptr(&pSql->param) == NULL) { + return; + } + assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; SSqlObj * pParObj = trs->pParentSql; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index c613c7ab9ad43f3aa2e3fe4b80adad13dd59ee82..51bf7bbaefe501f02be34f90003b29799bda5faa 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -106,6 +106,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" ) + +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" ) + + #ifndef TAOS_MESSAGE_C TSDB_MSG_TYPE_MAX // 105 #endif diff --git a/src/inc/twal.h b/src/inc/twal.h index 92204abd7d34a9ee2eebf1b74c2e8d58b9599f17..1ce7b132b068526f87fa594845bb591ade6f4966 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -22,22 +22,22 @@ extern "C" { #define TAOS_WAL_NOLOG 0 #define TAOS_WAL_WRITE 1 #define TAOS_WAL_FSYNC 2 - + typedef struct { - int8_t msgType; - int8_t reserved[3]; - int32_t len; - uint64_t version; - uint32_t signature; - uint32_t cksum; - char cont[]; + int8_t msgType; + int8_t reserved[3]; + int32_t len; + uint64_t version; + uint32_t signature; + uint32_t cksum; + char cont[]; } SWalHead; typedef struct { - int8_t walLevel; // wal level - int32_t fsyncPeriod; // millisecond - int8_t wals; // number of WAL files; - int8_t keep; // keep the wal file when closed + int8_t walLevel; // wal level + int32_t fsyncPeriod; // millisecond + int8_t wals; // number of WAL files; + int8_t keep; // keep the wal file when closed } SWalCfg; typedef void* twalh; // WAL HANDLE @@ -53,9 +53,6 @@ int walRestore(twalh, void *pVnode, FWalWrite writeFp); int walGetWalFile(twalh, char *name, uint32_t *index); int64_t walGetVersion(twalh); -extern int wDebugFlag; - - #ifdef __cplusplus } #endif diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 6cb7c669cc7a08434b2558588067d007b51b3595..2083ad3e9b7d5a101f35af4fe20dcd38957d5be2 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -80,7 +80,10 @@ int main(int argc, char* argv[]) { shellParseArgument(argc, argv, &args); if (args.netTestRole && args.netTestRole[0] != 0) { - taosNetTest(args.host, (uint16_t)args.port, (uint16_t)args.endPort, args.pktLen, args.netTestRole); + taos_init(); + CmdArguments cmdArgs; + memcpy(&cmdArgs, &args, sizeof(SShellArguments)); + taosNetTest(&cmdArgs); exit(0); } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 4e92581cc52d6b7540c32e19fc02aa6de6734dae..83dd80d5338a5b00b050500c9a6f873ef29e1762 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -663,13 +663,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 4c9493bc100d68b7751b71a8f6f149fc4941e5bc..1dfd87ff3a0e3d27f5c8091dd4ccee16f4971446 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1076,6 +1076,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); } } else { // msg is passed to app only parsing is ok + + if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS); + rpcFreeMsg(pRecv->msg); + return pConn; + } + rpcProcessIncomingMsg(pConn, pHead, pContext); } } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ebb6c3a0a9edff5acfc5f2ce7da8b58f03d8ab4a..19a5d3ba41b8fb0752f5849b8504059accdae485 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -56,6 +56,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { int code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; uint32_t pindex = 0; // index in last restore + bool fileChanged = false; *fversion = 0; sinfo.index = 0; @@ -114,10 +115,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { close(dfd); if (ret < 0) break; + fileChanged = true; sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } - if (code == 0 && (minfo.fversion != sinfo.fversion)) { + if (code == 0 && fileChanged) { // data file is changed, code shall be set to 1 *fversion = minfo.fversion; code = 1; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 626ad77da2eab4be9e94516c4e5c7c0e5a45837e..5633e97cdf56bea8f2c24fc862d720bea2c376db 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -79,7 +79,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { DIR * dir = NULL; int fid = 0; int vid = 0; - regex_t regex1, regex2; + regex_t regex1 = {0}, regex2 = {0}; int code = 0; char fname[TSDB_FILENAME_LEN] = "\0"; @@ -95,9 +95,27 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { dir = opendir(tDataDir); if (dir == NULL) { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (errno == ENOENT) { + tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir); + terrno = TAOS_SYSTEM_ERROR(errno); + + if (taosMkDir(tDataDir, 0755) < 0) { + tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + dir = opendir(tDataDir); + if (dir == NULL) { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } else { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } code = regcomp(®ex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED); diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 2179592f6d9416d0f4e1c2bdd300a06271425bec..78b9c90979fa74bdd0ce5ecfd53042db30e4e8fa 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,6 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h index 3fe1dfa9204fbbf85f193078b17e0bb4f9643848..426df5cbb28b9c0fcada049c7242730359d2a3fc 100644 --- a/src/util/inc/tnettest.h +++ b/src/util/inc/tnettest.h @@ -20,7 +20,27 @@ extern "C" { #endif -void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole); +typedef struct CmdArguments { + char* host; + char* password; + char* user; + char* auth; + char* database; + char* timezone; + bool is_raw_time; + bool is_use_passwd; + char file[TSDB_FILENAME_LEN]; + char dir[TSDB_FILENAME_LEN]; + int threadNum; + char* commands; + int abort; + int port; + int endPort; + int pktLen; + char* netTestRole; +} CmdArguments; + +void taosNetTest(CmdArguments* args); #ifdef __cplusplus } diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 3793f3d3a944cc5c8d86c0dc0c0fa5bfe3cee764..6fd526598365f831addd1bacb0b7f748d9552fdb 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -15,11 +15,16 @@ #include "os.h" #include "taosdef.h" +#include "taosmsg.h" #include "taoserror.h" #include "tulog.h" #include "tconfig.h" #include "tglobal.h" #include "tsocket.h" +#include "trpc.h" +#include "rpcHead.h" +#include "tutil.h" +#include "tnettest.h" #define MAX_PKG_LEN (64*1000) #define BUFFER_SIZE (MAX_PKG_LEN + 1024) @@ -30,9 +35,15 @@ typedef struct { uint16_t pktLen; } info_s; -static char serverFqdn[TSDB_FQDN_LEN]; +extern int tsRpcMaxUdpSize; + +static char g_user[TSDB_USER_LEN+1] = {0}; +static char g_pass[TSDB_PASSWORD_LEN+1] = {0}; +static char g_serverFqdn[TSDB_FQDN_LEN] = {0}; static uint16_t g_startPort = 0; static uint16_t g_endPort = 6042; +static uint32_t g_pktLen = 0; + static void *bindUdpPort(void *sarg) { info_s *pinfo = (info_s *)sarg; @@ -321,19 +332,145 @@ static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uin return ; } -static void taosNetTestClient(const char* serverFqdn, uint16_t startPort, uint16_t endPort, int pktLen) { - uint32_t serverIp = taosGetIpFromFqdn(serverFqdn); - if (serverIp == 0xFFFFFFFF) { - printf("Failed to resolve FQDN:%s", serverFqdn); - exit(-1); +void* tnetInitRpc(char* secretEncrypt, char spi) { + SRpcInit rpcInit; + void* pRpcConn = NULL; + + taosEncryptPass((uint8_t *)g_pass, strlen(g_pass), secretEncrypt); + + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "NET-TEST"; + rpcInit.numOfThreads = 1; // every DB connection has only one thread + rpcInit.cfp = NULL; + rpcInit.sessions = 16; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = g_user; + rpcInit.idleTime = 2000; + rpcInit.ckey = "key"; + rpcInit.spi = spi; + rpcInit.secret = secretEncrypt; + + pRpcConn = rpcOpen(&rpcInit); + return pRpcConn; +} + +static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi) { + SRpcEpSet epSet; + SRpcMsg reqMsg; + SRpcMsg rspMsg; + void* pRpcConn; + + char secretEncrypt[32] = {0}; + + pRpcConn = tnetInitRpc(secretEncrypt, spi); + if (NULL == pRpcConn) { + return -1; } - checkPort(serverIp, startPort, endPort, pktLen); + memset(&epSet, 0, sizeof(SRpcEpSet)); + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = port; + strcpy(epSet.fqdn[0], serverFqdn); + + reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; + reqMsg.pCont = rpcMallocCont(pktLen); + reqMsg.contLen = pktLen; + reqMsg.code = 0; + reqMsg.handle = NULL; // rpc handle returned to app + reqMsg.ahandle = NULL; // app handle set by client + + rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); + + // handle response + if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) { + //printf("code:%d[%s]\n", rspMsg.code, tstrerror(rspMsg.code)); + return -1; + } + + rpcFreeCont(rspMsg.pCont); - return; + rpcClose(pRpcConn); + + return 0; +} + +static void rpcCheckPort(uint32_t hostIp) { + int ret; + char spi; + + for (uint16_t port = g_startPort; port <= g_endPort; port++) { + //printf("test: %s:%d\n", info.host, port); + printf("\n"); + + //================ check tcp port ================ + int32_t pktLen; + if (g_pktLen <= tsRpcMaxUdpSize) { + pktLen = tsRpcMaxUdpSize + 1000; + } else { + pktLen = g_pktLen; + } + + spi = 1; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + spi = 0; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + printf("TCP port:%d test fail.\t\t", port); + } else { + //printf("tcp port:%d test ok.\t\t", port); + printf("TCP port:\033[32m%d test OK\033[0m\t\t", port); + } + } else { + //printf("tcp port:%d test ok.\t\t", port); + printf("TCP port:\033[32m%d test OK\033[0m\t\t", port); + } + + //================ check udp port ================ + if (g_pktLen >= tsRpcMaxUdpSize) { + pktLen = tsRpcMaxUdpSize - 1000; + } else { + pktLen = g_pktLen; + } + + spi = 0; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + spi = 1; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + printf("udp port:%d test fail.\t\n", port); + } else { + //printf("udp port:%d test ok.\t\n", port); + printf("UDP port:\033[32m%d test OK\033[0m\t\n", port); + } + } else { + //printf("udp port:%d test ok.\t\n", port); + printf("UDP port:\033[32m%d test OK\033[0m\t\n", port); + } + } + + printf("\n"); + return ; } +static void taosNetTestClient(int flag) { + uint32_t serverIp = taosGetIpFromFqdn(g_serverFqdn); + if (serverIp == 0xFFFFFFFF) { + printf("Failed to resolve FQDN:%s", g_serverFqdn); + exit(-1); + } + if (0 == flag) { + checkPort(serverIp, g_startPort, g_endPort, g_pktLen); + } else { + rpcCheckPort(serverIp); + } + + return; +} static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) { @@ -375,49 +512,66 @@ static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) } -void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole) { - if (pktLen > MAX_PKG_LEN) { - printf("test packet len overflow: %d, max len not greater than %d bytes\n", pktLen, MAX_PKG_LEN); - exit(-1); +void taosNetTest(CmdArguments *args) { + if (0 == args->pktLen) { + g_pktLen = 1000; + } else { + g_pktLen = args->pktLen; } - if (port && endPort) { - if (port > endPort) { - printf("endPort[%d] must not lesss port[%d]\n", endPort, port); + if (args->port && args->endPort) { + if (args->port > args->endPort) { + printf("endPort[%d] must not lesss port[%d]\n", args->endPort, args->port); exit(-1); } } - if (host && host[0] != 0) { - if (strlen(host) >= TSDB_EP_LEN) { - printf("host invalid: %s\n", host); + if (args->host && args->host[0] != 0) { + if (strlen(args->host) >= TSDB_EP_LEN) { + printf("host invalid: %s\n", args->host); exit(-1); } - taosGetFqdnPortFromEp(host, serverFqdn, &g_startPort); + taosGetFqdnPortFromEp(args->host, g_serverFqdn, &g_startPort); } else { - tstrncpy(serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN); + tstrncpy(g_serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN); g_startPort = tsServerPort; } - if (port) { - g_startPort = port; + if (args->port) { + g_startPort = args->port; } - if (endPort) { - g_endPort = endPort; + if (args->endPort) { + g_endPort = args->endPort; } - if (port > endPort) { + if (g_startPort > g_endPort) { printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort); exit(-1); } + + + if (args->is_use_passwd) { + if (args->password == NULL) args->password = getpass("Enter password: "); + } else { + args->password = TSDB_DEFAULT_PASS; + } + tstrncpy(g_pass, args->password, TSDB_PASSWORD_LEN); + + if (args->user == NULL) { + args->user = TSDB_DEFAULT_USER; + } + tstrncpy(g_user, args->user, TSDB_USER_LEN); - if (0 == strcmp("client", netTestRole)) { - printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", serverFqdn, g_startPort, g_endPort, pktLen); - taosNetTestClient(serverFqdn, g_startPort, g_endPort, pktLen); - } else if (0 == strcmp("server", netTestRole)) { - taosNetTestServer(g_startPort, g_endPort, pktLen); + if (0 == strcmp("client", args->netTestRole)) { + printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen); + taosNetTestClient(0); + } else if (0 == strcmp("clients", args->netTestRole)) { + printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen); + taosNetTestClient(1); + } else if (0 == strcmp("server", args->netTestRole)) { + taosNetTestServer(g_startPort, g_endPort, g_pktLen); } } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 100373e924b2ce3efa90ca84973077c8a259a799..3b3f0420b561d1d5371fca5a05999161b384a187 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -292,11 +292,15 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { if (pVnode->tsdb == NULL) { vnodeCleanUp(pVnode); return terrno; - } else if (terrno != TSDB_CODE_SUCCESS && pVnode->syncCfg.replica <= 1) { + } else if (terrno != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); - vnodeCleanUp(pVnode); - return terrno; + if (pVnode->syncCfg.replica <= 1) { + vnodeCleanUp(pVnode); + return terrno; + } else { + pVnode->version = 0; + } } sprintf(temp, "%s/wal", rootDir); diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h new file mode 100644 index 0000000000000000000000000000000000000000..593611589d6e1eb5d90a0eb8986f21bb32d5d1c6 --- /dev/null +++ b/src/wal/inc/walInt.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_WAL_INT_H +#define TDENGINE_WAL_INT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +extern int32_t wDebugFlag; + +#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} +#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} +#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} +#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} +#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} + +#define walPrefix "wal" +#define walSignature (uint32_t)(0xFAFBFDFE) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/wal/inc/walMgmt.h b/src/wal/inc/walMgmt.h new file mode 100644 index 0000000000000000000000000000000000000000..a23c7f8ec3d0064de23c03a62a81388b63b0f319 --- /dev/null +++ b/src/wal/inc/walMgmt.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_WAL_MGMT_H +#define TDENGINE_WAL_MGMT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c new file mode 100644 index 0000000000000000000000000000000000000000..2dd094d860528fa95084b7d3a0f404007cb757ed --- /dev/null +++ b/src/wal/src/walMgmt.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "twal.h" +#include "walInt.h" +#include "walMgmt.h" \ No newline at end of file diff --git a/src/wal/src/walMain.c b/src/wal/src/walWrite.c similarity index 95% rename from src/wal/src/walMain.c rename to src/wal/src/walWrite.c index 3492246a6f2610ba70874d534c2e9a34ac47c149..95587caa141a8964a845a9a0b47485bfc20fa998 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walWrite.c @@ -14,11 +14,10 @@ */ #define _DEFAULT_SOURCE - -#define TAOS_RANDOM_FILE_FAIL_TEST - #include "os.h" -#include "tlog.h" +#include "twal.h" +#include "walInt.h" +#include "walMgmt.h" #include "tchecksum.h" #include "tutil.h" #include "ttimer.h" @@ -26,14 +25,6 @@ #include "twal.h" #include "tqueue.h" -#define walPrefix "wal" - -#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} -#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} -#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} -#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} -#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} -#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} typedef struct { uint64_t version; @@ -54,7 +45,6 @@ typedef struct { static void *walTmrCtrl = NULL; static int tsWalNum = 0; static pthread_once_t walModuleInit = PTHREAD_ONCE_INIT; -static uint32_t walSignature = 0xFAFBFDFE; static int walHandleExistingFiles(const char *path); static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp); static int walRemoveWalFiles(const char *path); diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh new file mode 100755 index 0000000000000000000000000000000000000000..1a4c12a16c7d9bc4e8b2dd327765251745d44223 --- /dev/null +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log + +for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` +do +if [ -n "$memError" ]; then + if [ "$memError" -gt 12 ]; then + echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ + More than our threshold! ## ${NC}" + fi +fi +done + +grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log +for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'` +do + +if [ -n "$defiMemError" ]; then + if [ "$defiMemError" -gt 3 ]; then + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 + fi +fi +done \ No newline at end of file diff --git a/tests/script/general/db/nosuchfile.sim b/tests/script/general/db/nosuchfile.sim new file mode 100644 index 0000000000000000000000000000000000000000..98ac4ec012dc694357878a61ca0dbc11259f0a9e --- /dev/null +++ b/tests/script/general/db/nosuchfile.sim @@ -0,0 +1,66 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ========== step3 +sql create database d1 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 35) +sql insert into d1.t1 values(now+2s, 34) +sql insert into d1.t1 values(now+3s, 33) +sql insert into d1.t1 values(now+4s, 32) +sql insert into d1.t1 values(now+5s, 31) + +print ========== step4 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +sleep 3000 + +print ========== step5 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +print ========== step6 +system_content rm -rf ../../../sim/dnode1/data/vnode/vnode2/tsdb/data + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != null then + return -1 +endi +if $data11 != null then + return -1 +endi +if $data21 != null then + return -1 +endi +if $data31 != null then + return -1 +endi +if $data41 != null then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e68d1566a76468b06115a80aace70871665c9cb..1b2fe37c71f26486ba847006ecf3aa373b5f55c1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -64,6 +64,7 @@ cd ../../../debug; make ./test.sh -f general/db/repeat.sim ./test.sh -f general/db/tables.sim ./test.sh -f general/db/vnodes.sim +./test.sh -f general/db/nosuchfile.sim ./test.sh -f general/field/2.sim ./test.sh -f general/field/3.sim @@ -277,6 +278,7 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balance2.sim ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/data1.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim ./test.sh -f unique/dnode/reason.sim diff --git a/tests/script/unique/dnode/data1.sim b/tests/script/unique/dnode/data1.sim new file mode 100644 index 0000000000000000000000000000000000000000..61a991148b21b471aef906223a5f600f7db38f5f --- /dev/null +++ b/tests/script/unique/dnode/data1.sim @@ -0,0 +1,137 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ========== step2 +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +show2: + $x = $x + 1 + sleep 3000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + goto show2 +endi +if $data2_2 != 0 then + goto show2 +endi +if $data2_3 != 0 then + goto show2 +endi +if $data2_4 != 0 then + goto show2 +endi + +print ========== step3 +sql create database d1 replica 3 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 35) +sql insert into d1.t1 values(now+2s, 34) +sql insert into d1.t1 values(now+3s, 33) +sql insert into d1.t1 values(now+4s, 32) +sql insert into d1.t1 values(now+5s, 31) + +$x = 0 +show3: + $x = $x + 1 + sleep 3000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + goto show3 +endi +if $data2_2 != 1 then + goto show3 +endi +if $data2_3 != 1 then + goto show3 +endi +if $data2_4 != 1 then + goto show3 +endi + +print ========== step4 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +print ========== step5 +system_content rm -rf ../../../sim/dnode4/data/vnode/vnode2/tsdb/data + +print ========== step6 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sleep 10000 + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file