Merge branch 'develop' of https://github.com/taosdata/TDengine into develop

src/client/src/tscSQLParser.c

@@ -4642,21 +4642,24 @@ typedef struct SDNodeDynConfOption {
 } SDNodeDynConfOption;
 
 
-int32_t validateEp(char* ep) {
+int32_t validateEp(char* ep) {  
   char buf[TSDB_EP_LEN + 1] = {0};
   tstrncpy(buf, ep, TSDB_EP_LEN);
 
-  char *pos = strchr(buf, ':');
-  if (NULL == pos) {   
-    return TSDB_CODE_TSC_INVALID_SQL;
+  char* pos = strchr(buf, ':');
+  if (NULL == pos) {
+    int32_t val = strtol(ep, NULL, 10);
+    if (val <= 0 || val > 65536) {
+      return TSDB_CODE_TSC_INVALID_SQL;
+    }
+  } else {
+    uint16_t port = atoi(pos + 1);
+    if (0 == port) {
+      return TSDB_CODE_TSC_INVALID_SQL;
+    }
   }
-  
-  uint16_t port = atoi(pos+1);
-  if (0 == port) {   
-    return TSDB_CODE_TSC_INVALID_SQL;
-  }  
 
-  return TSDB_CODE_SUCCESS;   
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t validateDNodeConfig(tDCLSQL* pOptions) {
@@ -4664,13 +4667,13 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
 
-  const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 17;
+  const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 19;
   const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[] = {
       {"resetLog", 8},       {"resetQueryCache", 15},  {"debugFlag", 9},     {"mDebugFlag", 10},
       {"dDebugFlag", 10},    {"sdbDebugFlag", 12},     {"vDebugFlag", 10},   {"cDebugFlag", 10},
       {"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"rpcDebugFlag", 12}, {"uDebugFlag", 10},
       {"tmrDebugFlag", 12},  {"qDebugflag", 10},       {"sDebugflag", 10},   {"tsdbDebugFlag", 13},
-      {"monitor", 7}};
+      {"mqttDebugFlag", 13}, {"wDebugFlag", 10},       {"monitor", 7}};
 
   SSQLToken* pOptionToken = &pOptions->a[1];
 
@@ -4694,7 +4697,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
     SSQLToken* pValToken = &pOptions->a[2];
 
     int32_t val = strtol(pValToken->z, NULL, 10);
-    if (val < 131 || val > 199) {
+    if (val < 0 || val > 256) {
       /* options value is out of valid range */
       return TSDB_CODE_TSC_INVALID_SQL;
     }

src/dnode/src/dnodeMgmt.c

@@ -706,6 +706,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
 
   // fill cluster cfg parameters
   pStatus->clusterCfg.numOfMnodes        = htonl(tsNumOfMnodes);
+  pStatus->clusterCfg.enableBalance      = htonl(tsEnableBalance);
   pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
   pStatus->clusterCfg.offlineThreshold   = htonl(tsOfflineThreshold);
   pStatus->clusterCfg.statusInterval     = htonl(tsStatusInterval);

src/inc/taosmsg.h

@@ -563,15 +563,16 @@ typedef struct {
 
 typedef struct {
   int32_t  numOfMnodes;               // tsNumOfMnodes
+  int32_t  enableBalance;             // tsEnableBalance
   int32_t  mnodeEqualVnodeNum;        // tsMnodeEqualVnodeNum
   int32_t  offlineThreshold;          // tsOfflineThreshold
   int32_t  statusInterval;            // tsStatusInterval
+  int32_t  maxtablesPerVnode;
+  int32_t  maxVgroupsPerDb;
   char     arbitrator[TSDB_EP_LEN];   // tsArbitrator
   char     timezone[64];              // tsTimezone
   char     locale[TSDB_LOCALE_LEN];   // tsLocale
   char     charset[TSDB_LOCALE_LEN];  // tsCharset
-  int32_t  maxtablesPerVnode;
-  int32_t  maxVgroupsPerDb;
 } SClusterCfg;
 
 typedef struct {

src/mnode/src/mnodeDnode.c

@@ -269,18 +269,37 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) {
 }
 
 static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
+  if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
+    mError("failed to cfg dnode, no rights");
+    return TSDB_CODE_MND_NO_RIGHTS;
+  }
+  
   SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont;
   if (pCmCfgDnode->ep[0] == 0) {
-    strcpy(pCmCfgDnode->ep, tsLocalEp);
-  } else {
-    // TODO temporary disabled for compiling: strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep); 
+    tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN);
+  } 
+
+  int32_t dnodeId = 0;
+  char* pos = strchr(pCmCfgDnode->ep, ':');
+  if (NULL == pos) {
+    dnodeId = strtol(pCmCfgDnode->ep, NULL, 10);
+    if (dnodeId <= 0 || dnodeId > 65536) {
+      mError("failed to cfg dnode, invalid dnodeId:%s", pCmCfgDnode->ep);
+      return TSDB_CODE_MND_DNODE_NOT_EXIST;
+    }
   }
 
-  if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
-    return TSDB_CODE_MND_NO_RIGHTS;
+  SRpcIpSet ipSet = mnodeGetIpSetFromIp(pCmCfgDnode->ep);
+  if (dnodeId != 0) {
+    SDnodeObj *pDnode = mnodeGetDnode(dnodeId);
+    if (pDnode == NULL) {
+      mError("failed to cfg dnode, invalid dnodeId:%d", dnodeId);
+      return TSDB_CODE_MND_DNODE_NOT_EXIST;
+    }
+    ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp);
+    mnodeDecDnodeRef(pDnode);
   }
 
-  SRpcIpSet ipSet = mnodeGetIpSetFromIp(pCmCfgDnode->ep);
   SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg));
   strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep);
   strcpy(pMdCfgDnode->config, pCmCfgDnode->config);
@@ -292,9 +311,9 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
     .pCont = pMdCfgDnode,
     .contLen = sizeof(SMDCfgDnodeMsg)
   };
-  dnodeSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg);
 
   mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user);
+  dnodeSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -305,6 +324,7 @@ static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
 
 static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
   if (clusterCfg->numOfMnodes        != htonl(tsNumOfMnodes))        return false;
+  if (clusterCfg->enableBalance      != htonl(tsEnableBalance))        return false;
   if (clusterCfg->mnodeEqualVnodeNum != htonl(tsMnodeEqualVnodeNum)) return false;
   if (clusterCfg->offlineThreshold   != htonl(tsOfflineThreshold))   return false;
   if (clusterCfg->statusInterval     != htonl(tsStatusInterval))     return false;

src/mnode/src/mnodeVgroup.c

@@ -593,7 +593,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "maxTables");
+  strcpy(pSchema[cols].name, "onlineVnodes");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -692,8 +692,15 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
     *(int32_t *)pWrite = taosIdPoolMaxSize(pVgroup->idPool);
     cols++;
 
+    int32_t onlineVnodes = 0;
+    for (int32_t i = 0; i < pShow->maxReplica; ++i) {
+      if (pVgroup->vnodeGid[i].role == TAOS_SYNC_ROLE_SLAVE || pVgroup->vnodeGid[i].role == TAOS_SYNC_ROLE_MASTER) {
+        onlineVnodes++;
+      }
+    }
+
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = tsMaxTablePerVnode;
+    *(int32_t *)pWrite = onlineVnodes;
     cols++;
 
     for (int32_t i = 0; i < pShow->maxReplica; ++i) {

tests/pytest/fulltest.sh

@@ -143,6 +143,7 @@ python3 ./test.py -f query/filterOtherTypes.py
 python3 ./test.py -f query/querySort.py
 python3 ./test.py -f query/queryJoin.py
 python3 ./test.py -f query/select_last_crash.py
+python3 ./test.py -f query/queryNullValueTest.py
 
 #stream
 python3 ./test.py -f stream/metric_1.py

tests/pytest/query/queryInsertValue.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        
+        self.numOfRecords = 10
+        self.ts = 1537146000000    
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.execute("use db")
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+
+        tdSql.execute(
+            "create table st (ts timestamp, speed int) tags(areaid int, loc nchar(20))")
+        tdSql.execute("create table t1 using st tags(1, 'beijing')")
+        tdSql.execute("insert into t1 values(now, 1)")
+        tdSql.query("select * from st")
+        tdSql.checkRows(1)
+
+        tdSql.execute("alter table st add column length int")
+        tdSql.execute("insert into t1 values(now, 1, 2)")
+        tdSql.query("select last(*) from st")
+        tdSql.checkData(0, 2, 2);
+
+        self.restartTaosd();
+
+        tdSql.query("select last(*) from st")
+        tdSql.checkData(0, 2, 2);
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/query/queryNullValueTest.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        
+        self.numOfRecords = 10
+        self.ts = 1537146000000
+
+    def checkNullValue(self, result):
+        mx = np.array(result)
+        [rows, cols] = mx.shape
+        for i in range(rows):
+            for j in range(cols):
+                if j + 1 < cols and mx[i, j + 1] is not None:
+                    print(mx[i, j + 1])
+                    return False
+        return True
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.execute("use db")
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+
+        tdSql.execute(
+            "create table meters (ts timestamp, col1 int) tags(tgcol1 int)")
+        tdSql.execute("create table t0 using meters tags(NULL)")
+
+        for i in range (self.numOfRecords):
+            tdSql.execute("insert into t0 values (%d, %d)" % (self.ts + i, i));
+
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+
+        tdSql.execute("alter table meters add column col2 tinyint")
+        tdSql.execute("alter table meters drop column col1")        
+        tdSql.query("select * from meters")        
+        tdSql.checkRows(10)
+        tdSql.query("select col2 from meters")        
+        tdSql.checkRows(10)        
+
+        tdSql.execute("alter table meters add column col1 int")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select col1 from meters")        
+        tdSql.checkRows(10)
+
+        tdSql.execute("alter table meters add column col3 smallint")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select col3 from meters")        
+        tdSql.checkRows(10)
+
+        tdSql.execute("alter table meters add column col4 bigint")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select col4 from meters")        
+        tdSql.checkRows(10)
+
+        tdSql.execute("alter table meters add column col5 float")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select col5 from meters")        
+        tdSql.checkRows(10)
+
+        tdSql.execute("alter table meters add column col6 double")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select col6 from meters")        
+        tdSql.checkRows(10)
+
+        tdSql.execute("alter table meters add column col7 bool")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select col7 from meters")        
+        tdSql.checkRows(10)
+
+        tdSql.execute("alter table meters add column col8 binary(20)")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select col8 from meters")        
+        tdSql.checkRows(10)
+
+        tdSql.execute("alter table meters add column col9 nchar(20)")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select col9 from meters")        
+        tdSql.checkRows(10)
+        
+        tdSql.execute("alter table meters add tag tgcol2 tinyint")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select tgcol2 from meters")
+        tdSql.checkRows(1)
+                
+
+        tdSql.execute("alter table meters add tag tgcol3 smallint")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select tgcol3 from meters")
+        tdSql.checkRows(1)
+        
+
+        tdSql.execute("alter table meters add tag tgcol4 bigint")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select tgcol4 from meters")
+        tdSql.checkRows(1)
+
+        tdSql.execute("alter table meters add tag tgcol5 float")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select tgcol5 from meters")
+        tdSql.checkRows(1)
+
+        tdSql.execute("alter table meters add tag tgcol6 double")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select tgcol6 from meters")
+        tdSql.checkRows(1)
+
+        tdSql.execute("alter table meters add tag tgcol7 bool")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select tgcol7 from meters")
+        tdSql.checkRows(1)
+
+        tdSql.execute("alter table meters add tag tgcol8 binary(20)")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select tgcol8 from meters")
+        tdSql.checkRows(1)
+
+        tdSql.execute("alter table meters add tag tgcol9 nchar(20)")
+        tdSql.query("select * from meters")
+        tdSql.checkRows(10)
+        tdSql.query("select tgcol9 from meters")
+        tdSql.checkRows(1)
+
+        self.restartTaosd()
+        tdSql.query("select * from meters")        
+        tdSql.checkRows(10)
+        if self.checkNullValue(tdSql.queryResult) is False:
+            tdLog.exit("non None value is detected")
+
+
+        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/regressiontest.sh

@@ -140,6 +140,7 @@ python3 ./test.py -f query/queryJoin.py
 python3 ./test.py -f query/filterCombo.py
 python3 ./test.py -f query/queryNormal.py
 python3 ./test.py -f query/select_last_crash.py
+python3 ./test.py -f query/queryNullValueTest.py
 
 #stream
 python3 ./test.py -f stream/stream1.py