Merge pull request #2943 from taosdata/develop

Mergre from develop into hotfix/test

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -97,7 +97,7 @@ public class TSDBJNIConnector {
 
         this.taos = this.connectImp(host, port, dbName, user, password);
         if (this.taos == TSDBConstants.JNI_NULL_POINTER) {
-            throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(null)), "", this.getErrCode(null));
+            throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(0L)), "", this.getErrCode(0l));
         }
 
         return true;
@@ -115,7 +115,7 @@ public class TSDBJNIConnector {
             freeResultSet(taosResultSetPointer);
         }
 
-        long pSql = 0l;
+        Long pSql = 0l;
         try {
             pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
         } catch (Exception e) {
@@ -124,16 +124,11 @@ public class TSDBJNIConnector {
             throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding"));
         }
         int code = this.getErrCode(pSql);
-
-        if (code < 0) {
+        if (code != 0) {
             affectedRows = -1;
-            if (code == TSDBConstants.JNI_TDENGINE_ERROR) {
-                this.freeResultSet(pSql);
-                throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(pSql)), "", this.getErrCode(pSql));
-            } else {
-                this.freeResultSet(pSql);
-                throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(pSql));
-            }
+            String err_msg = this.getErrMsg(pSql);
+            this.freeResultSet(pSql);
+            throw new SQLException(TSDBConstants.WrapErrMsg(err_msg), "", code);
         }
 
         // Try retrieving result set for the executed SQL using the current connection pointer. If the executed
@@ -151,20 +146,20 @@ public class TSDBJNIConnector {
     /**
      * Get recent error code by connection
      */
-    public int getErrCode(Long pSql) {
-        return Math.abs(this.getErrCodeImp(this.taos, pSql));
+    public int getErrCode(long pSql) {
+        return this.getErrCodeImp(this.taos, pSql);
     }
 
-    private native int getErrCodeImp(long connection, Long pSql);
+    private native int getErrCodeImp(long connection, long pSql);
 
     /**
      * Get recent error message by connection
      */
-    public String getErrMsg(Long pSql) {
-        return this.getErrMsgImp(this.taos, pSql);
+    public String getErrMsg(long pSql) {
+        return this.getErrMsgImp(pSql);
     }
 
-    private native String getErrMsgImp(long connection, Long pSql);
+    private native String getErrMsgImp(long pSql);
 
     /**
      * Get resultset pointer
@@ -248,7 +243,7 @@ public class TSDBJNIConnector {
     public void closeConnection() throws SQLException {
         int code = this.closeConnectionImp(this.taos);
         if (code < 0) {
-            throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(null));
+            throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(0l));
         } else if (code == 0) {
             this.taos = TSDBConstants.JNI_NULL_POINTER;
         } else {

src/dnode/src/dnodeTelemetry.c

@@ -22,8 +22,16 @@
 #include "tsocket.h"
 #include "tbuffer.h"
 #include "mnode.h"
+#include "mnodeDef.h"
+#include "mnodeDb.h"
+#include "mnodeDnode.h"
 #include "mnodeCluster.h"
+#include "mnodeDnode.h"
+#include "mnodeVgroup.h"
+#include "mnodeMnode.h"
+#include "mnodeTable.h"
 #include "mnodeSdb.h"
+#include "mnodeAcct.h"
 #include "dnode.h"
 #include "dnodeInt.h"
 #include "dnodeTelemetry.h"
@@ -170,18 +178,23 @@ static void addVersionInfo(SBufferWriter* bw) {
   addStringField(bw, "version", version);
   addStringField(bw, "buildInfo", buildinfo);
   addStringField(bw, "gitInfo", gitinfo);
-  //addStringField(&bw, "installAt", "2020-08-01T00:00:00Z");
 }
 
 static void addRuntimeInfo(SBufferWriter* bw) {
-  // addIntField(&bw, "numOfDnode", 1);
-  // addIntField(&bw, "numOfVnode", 1);
-  // addIntField(&bw, "numOfStable", 1);
-  // addIntField(&bw, "numOfTable", 1);
-  // addIntField(&bw, "numOfRows", 1);
-  // addStringField(&bw, "startAt", "2020-08-01T00:00:00Z");
-  // addStringField(&bw, "memoryUsage", "10240 kB");
-  // addStringField(&bw, "diskUsage", "10240 MB");
+  addIntField(bw, "numOfDnode", mnodeGetDnodesNum());
+  addIntField(bw, "numOfMnode", mnodeGetMnodesNum());
+  addIntField(bw, "numOfVgroup", mnodeGetVgroupNum());
+  addIntField(bw, "numOfDatabase", mnodeGetDbNum());
+  addIntField(bw, "numOfSuperTable", mnodeGetSuperTableNum());
+  addIntField(bw, "numOfChildTable", mnodeGetChildTableNum());
+
+  SAcctInfo info;
+  mnodeGetStatOfAllAcct(&info);
+  addIntField(bw, "numOfColumn", info.numOfTimeSeries);
+  addIntField(bw, "numOfPoint", info.totalPoints);
+  addIntField(bw, "totalStorage", info.totalStorage);
+  addIntField(bw, "compStorage", info.compStorage);
+  // addStringField(bw, "installTime", "2020-08-01T00:00:00Z");
 }
 
 static void sendTelemetryReport() {
@@ -230,18 +243,13 @@ static void sendTelemetryReport() {
 static void* telemetryThread(void* param) {
   struct timespec end = {0};
   clock_gettime(CLOCK_REALTIME, &end);
-  end.tv_sec += 300; // wait 5 minutes to send first report
+  end.tv_sec += 300; // wait 5 minutes before send first report
 
   while (1) {
-    while (1) {
-      if (sem_timedwait(&tsExitSem, &end) == 0) {
-        return NULL;
-      }
-      struct timespec now = {0};
-      clock_gettime(CLOCK_REALTIME, &now);
-      if (now.tv_sec > end.tv_sec || (now.tv_sec == end.tv_sec && now.tv_nsec >= end.tv_nsec)) {
-        break;
-      }
+    if (sem_timedwait(&tsExitSem, &end) == 0) {
+      break;
+    } else if (errno != ETIMEDOUT) {
+      continue;
     }
 
     if (sdbIsMaster()) {

src/kit/taosdemo/taosdemo.c

@@ -43,9 +43,10 @@ extern char configDir[];
 #define MAX_DB_NAME_SIZE 64
 #define MAX_TB_NAME_SIZE 64
 #define MAX_DATA_SIZE    1024
-#define MAX_NUM_DATATYPE 8
+#define MAX_NUM_DATATYPE 30
 #define OPT_ABORT        1 /* –abort */
 #define STRING_LEN       512
+#define MAX_PREPARED_RAND 1000000
 
 /* The options we understand. */
 static struct argp_option options[] = {
@@ -175,6 +176,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
           }
           sptr[index++] = token;
           token = strsep(&running, ", ");
+          if (index >= MAX_NUM_DATATYPE) break;
         }
       }
       break;
@@ -311,6 +313,8 @@ int generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp
 
 void rand_string(char *str, int size);
 
+void init_rand_data();
+
 double getCurrentTime();
 
 void callBack(void *param, TAOS_RES *res, int code);
@@ -361,7 +365,7 @@ int main(int argc, char *argv[]) {
   arguments.num_of_DPT = 100000;
   arguments.num_of_RPR = 1000;
   arguments.use_metric = true;
-  arguments.insert_only = false;
+  arguments.insert_only = true;
   // end change
 
   argp_parse(&argp, argc, argv, 0, 0, &arguments);
@@ -403,6 +407,7 @@ int main(int argc, char *argv[]) {
     taos_close(qtaos);
     return 0;
   }
+  init_rand_data();
 
   memset(dataString, 0, STRING_LEN);
   int len = 0;
@@ -411,7 +416,7 @@ int main(int argc, char *argv[]) {
     do_aggreFunc = false;
   }
   for (; count_data_type <= MAX_NUM_DATATYPE; count_data_type++) {
-    if (strcasecmp(data_type[count_data_type], "") == 0) {
+    if (data_type[count_data_type] == NULL) {
       break;
     }
 
@@ -1169,6 +1174,66 @@ double getCurrentTime() {
   return tv.tv_sec + tv.tv_usec / 1E6;
 }
 
+int32_t  randint[MAX_PREPARED_RAND];
+int64_t  randbigint[MAX_PREPARED_RAND];
+float  randfloat[MAX_PREPARED_RAND];
+double  randdouble[MAX_PREPARED_RAND];
+
+int32_t rand_tinyint(){
+  static int cursor;
+  cursor++;
+  cursor = cursor % MAX_PREPARED_RAND;
+  return randint[cursor] % 128;
+
+}
+
+int32_t rand_smallint(){
+  static int cursor;
+  cursor++;
+  cursor = cursor % MAX_PREPARED_RAND;
+  return randint[cursor] % 32767;
+}
+
+int32_t rand_int(){
+  static int cursor;
+  cursor++;
+  cursor = cursor % MAX_PREPARED_RAND;
+  return randint[cursor];
+}
+
+int64_t rand_bigint(){
+  static int cursor;
+  cursor++;
+  cursor = cursor % MAX_PREPARED_RAND;
+  return randbigint[cursor];
+  
+}
+
+float rand_float(){
+  static int cursor;
+  cursor++;
+  cursor = cursor % MAX_PREPARED_RAND;
+  return randfloat[cursor];  
+  
+}
+
+double rand_double() {
+  static int cursor;
+  cursor++;
+  cursor = cursor % MAX_PREPARED_RAND;
+  return randdouble[cursor];
+
+}
+
+void init_rand_data(){
+  for (int i = 0; i < MAX_PREPARED_RAND; i++){
+    randint[i] = (int)(rand() % 10);
+    randbigint[i] = (int64_t)(rand() % 2147483648);
+    randfloat[i] = (float)(rand() / 1000.0);
+    randdouble[i] = (double)(rand() / 1000000.0);
+  }
+}
+
 int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) {
   memset(res, 0, MAX_DATA_SIZE);
   char *pstr = res;
@@ -1176,7 +1241,7 @@ int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t times
   int c = 0;
 
   for (; c < MAX_NUM_DATATYPE; c++) {
-    if (strcasecmp(data_type[c], "") == 0) {
+    if (data_type[c] == NULL) {
       break;
     }
   }
@@ -1188,17 +1253,17 @@ int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t times
 
   for (int i = 0; i < num_of_cols; i++) {
     if (strcasecmp(data_type[i % c], "tinyint") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(rand() % 128));
+      pstr += sprintf(pstr, ", %d", rand_tinyint() );
     } else if (strcasecmp(data_type[i % c], "smallint") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(rand() % 32767));
+      pstr += sprintf(pstr, ", %d", rand_smallint());
     } else if (strcasecmp(data_type[i % c], "int") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(rand() % 10)); 
+      pstr += sprintf(pstr, ", %d", rand_int()); 
     } else if (strcasecmp(data_type[i % c], "bigint") == 0) {
-      pstr += sprintf(pstr, ", %" PRId64, rand() % 2147483648);
+      pstr += sprintf(pstr, ", %" PRId64, rand_bigint());
     } else if (strcasecmp(data_type[i % c], "float") == 0) {
-      pstr += sprintf(pstr, ", %10.4f", (float)(rand() / 1000.0));
+      pstr += sprintf(pstr, ", %10.4f", rand_float());
     } else if (strcasecmp(data_type[i % c], "double") == 0) {
-      double t = (double)(rand() / 1000000.0);
+      double t = rand_double();
       pstr += sprintf(pstr, ", %20.8f", t);
     } else if (strcasecmp(data_type[i % c], "bool") == 0) {
       bool b = rand() & 1;

src/mnode/inc/mnodeAcct.h

@@ -24,6 +24,7 @@ extern "C" {
 
 int32_t mnodeInitAccts();
 void    mnodeCleanupAccts();
+void    mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo);
 void *  mnodeGetAcct(char *acctName);
 void *  mnodeGetNextAcct(void *pIter, SAcctObj **pAcct);
 void    mnodeIncAcctRef(SAcctObj *pAcct);

src/mnode/inc/mnodeDb.h

@@ -30,6 +30,7 @@ enum _TSDB_DB_STATUS {
 // api
 int32_t mnodeInitDbs();
 void    mnodeCleanupDbs();
+int64_t mnodeGetDbNum();
 SDbObj *mnodeGetDb(char *db);
 SDbObj *mnodeGetDbByTableId(char *db);
 void *  mnodeGetNextDb(void *pIter, SDbObj **pDb);

src/mnode/inc/mnodeTable.h

@@ -24,6 +24,8 @@ extern "C" {
 
 int32_t mnodeInitTables();
 void    mnodeCleanupTables();
+int64_t mnodeGetSuperTableNum();
+int64_t mnodeGetChildTableNum();
 void *  mnodeGetTable(char *tableId);
 void    mnodeIncTableRef(void *pTable);
 void    mnodeDecTableRef(void *pTable);

src/mnode/inc/mnodeVgroup.h

@@ -24,6 +24,7 @@ struct SMnodeMsg;
 
 int32_t mnodeInitVgroups();
 void    mnodeCleanupVgroups();
+int64_t mnodeGetVgroupNum();
 SVgObj *mnodeGetVgroup(int32_t vgId);
 void    mnodeIncVgroupRef(SVgObj *pVgroup);
 void    mnodeDecVgroupRef(SVgObj *pVgroup);

src/mnode/src/mnodeAcct.c

@@ -23,6 +23,7 @@
 #include "mnodeDb.h"
 #include "mnodeSdb.h"
 #include "mnodeUser.h"
+#include "mnodeVgroup.h"
 
 #include "tglobal.h"
 
@@ -130,6 +131,37 @@ void mnodeCleanupAccts() {
   tsAcctSdb = NULL;
 }
 
+void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) {
+  memset(pAcctInfo, 0, sizeof(*pAcctInfo));
+
+  void   *pIter = NULL;
+  SAcctObj *pAcct = NULL;
+  while (1) {
+    pIter = mnodeGetNextAcct(pIter, &pAcct);
+    if (pAcct == NULL) {
+      break;
+    }
+    pAcctInfo->numOfDbs += pAcct->acctInfo.numOfDbs;
+    pAcctInfo->numOfTimeSeries += pAcct->acctInfo.numOfTimeSeries;
+    mnodeDecAcctRef(pAcct);
+  }
+  sdbFreeIter(pIter);
+
+  SVgObj *pVgroup = NULL;
+  pIter = NULL;
+  while (1) {
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) {
+       break;
+    }
+    pAcctInfo->totalStorage += pVgroup->totalStorage;
+    pAcctInfo->compStorage += pVgroup->compStorage;
+    pAcctInfo->totalPoints += pVgroup->pointsWritten;
+    mnodeDecVgroupRef(pVgroup);
+  }
+  sdbFreeIter(pIter);
+}
+
 void *mnodeGetAcct(char *name) {
   return sdbGetRow(tsAcctSdb, name);
 }

src/mnode/src/mnodeDb.c

@@ -61,6 +61,10 @@ static int32_t mnodeDbActionDestroy(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
+int64_t mnodeGetDbNum() {
+  return sdbGetNumOfRows(tsDbSdb);
+}
+
 static int32_t mnodeDbActionInsert(SSdbOper *pOper) {
   SDbObj *pDb = pOper->pObj;
   SAcctObj *pAcct = mnodeGetAcct(pDb->acct);

src/mnode/src/mnodeTable.c

@@ -375,6 +375,14 @@ static void mnodeCleanupChildTables() {
   tsChildTableSdb = NULL;
 }
 
+int64_t mnodeGetSuperTableNum() {
+  return sdbGetNumOfRows(tsSuperTableSdb);
+}
+
+int64_t mnodeGetChildTableNum() {
+  return sdbGetNumOfRows(tsChildTableSdb);
+}
+
 static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
   atomic_add_fetch_32(&pStable->numOfTables, 1);
 

src/mnode/src/mnodeVgroup.c

@@ -605,6 +605,10 @@ void mnodeCleanupVgroups() {
   tsVgroupSdb = NULL;
 }
 
+int64_t mnodeGetVgroupNum() {
+  return sdbGetNumOfRows(tsVgroupSdb);
+}
+
 static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SDbObj *pDb = mnodeGetDb(pShow->db);
   if (pDb == NULL) {

tests/perftest-scripts/coverage_test.sh

@@ -32,7 +32,14 @@ function buildTDengine {
 		echo "repo need to pull"
     git reset --hard
 		git pull
+  fi
 
+  [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
+  cd $TDENGINE_DIR/debug
+  [ -f $TDENGINE_DIR/debug/build/bin/taosd ] || need_rebuild=true
+
+  if $need_rebuild ; then
+    echo "rebuild.."
 		LOCAL_COMMIT=`git rev-parse --short @`
 
 		rm -rf *
@@ -46,10 +53,9 @@ function buildTDengine {
 function runGeneralCaseOneByOne {
 	while read -r line; do
 		if [[ $line =~ ^./test.sh* ]]; then
-			general_case=`echo $line | grep -w general`
+			case=`echo $line | grep -w "general\|unique\/mnode\/mgmt33.sim\|unique\/stable\/dnode3.sim\|unique\/cluster\/balance3.sim\|unique\/arbitrator\/offline_replica2_alterTable_online.sim"|awk '{print $NF}'`
 
-			if [ -n "$general_case" ]; then
-				case=`echo $line |grep general| awk '{print $NF}'`
+			if [ -n "$case" ]; then
 				./test.sh -f $case > /dev/null 2>&1 && \
 				echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \
 				echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
@@ -90,14 +96,14 @@ function runTest {
 	sleep 10
 
 	cd $TDENGINE_DIR/src/connector/jdbc
-	mvn clean package
-	mvn test | tee -a $TDENGINE_COVERAGE_REPORT
+	mvn clean package > /dev/null 2>&1
+	mvn test > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
 
 	# Test C Demo
 	stopTaosd
 	$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
 	sleep 10
-	yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDENGINE_COVERAGE_REPORT
+	yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
 
 	# Test waltest
 	dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
@@ -105,14 +111,14 @@ function runTest {
 	echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT
 	echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT
 	if [ -n "$walDir" ]; then
-		yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDENGINE_COVERAGE_REPORT
+		yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
 	fi
 
   # run Unit Test
   echo "Run Unit Test: utilTest, queryTest and cliTest"
-  $TDENGINE_DIR/debug/build/bin/utilTest > /dev/null && echo "utilTest pass!" || echo "utilTest failed!"
-  $TDENGINE_DIR/debug/build/bin/queryTest > /dev/null && echo "queryTest pass!" || echo "queryTest failed!"
-  $TDENGINE_DIR/debug/build/bin/cliTest > /dev/null && echo "cliTest pass!" || echo "cliTest failed!"
+  $TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!"
+  $TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!"
+  $TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!"
 
 	stopTaosd
 }

tests/perftest-scripts/tdinternal_coverage_test.sh

@@ -100,23 +100,30 @@ function runTest {
 	sleep 10
 
 	cd $TDINTERNAL_DIR/community/src/connector/jdbc
-	mvn clean package
-	mvn test | tee -a $TDINTERNAL_COVERAGE_REPORT
+	mvn clean package > /dev/null
+	mvn test > /dev/null | tee -a $TDINTERNAL_COVERAGE_REPORT
 
 	# Test C Demo
 	stopTaosd
 	$TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null &
 	sleep 10
-	yes | $TDINTERNAL_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDINTERNAL_COVERAGE_REPORT
+	yes | $TDINTERNAL_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null | tee -a $TDINTERNAL_COVERAGE_REPORT
 
 	# Test waltest
 	dataDir=`grep dataDir $TDINTERNAL_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
 	walDir=`find $dataDir -name "wal"|head -n1`
-	echo "dataDir: $dataDir\nwalDir: $walDir" | tee -a $TDINTERNAL_COVERAGE_REPORT
+	echo "dataDir: $dataDir" | tee -a $TDINTERNAL_COVERAGE_REPORT
+  echo "walDir: $walDir" | tee -a $TDINTERNAL_COVERAGE_REPORT
 	if [ -n "$walDir" ]; then
-		yes | $TDINTERNAL_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDINTERNAL_COVERAGE_REPORT
+		yes | $TDINTERNAL_DIR/debug/build/bin/waltest -p $walDir > /dev/null | tee -a $TDINTERNAL_COVERAGE_REPORT
 	fi
 
+  # run Unit Test
+  echo "Run Unit Test: utilTest, queryTest and cliTest"
+  $TDINTERNAL_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!"
+  $TDINTERNAL_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!"
+  $TDINTERNAL_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!"
+
 	stopTaosd
 }