Merge branch 'develop' into feature/2.0tsdb

.gitignore

@@ -11,6 +11,7 @@ debs/
 rpms/
 mac/
 *.pyc
+.mypy_cache
 *.tmp
 *.swp
 src/connector/nodejs/node_modules/
@@ -38,3 +39,5 @@ pysim/
 *.out
 *DS_Store
 
+# Doxygen Generated files
+html/

src/client/src/tscParseInsert.c

@@ -779,6 +779,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
 
     STagData *pTag = (STagData *)pCmd->payload;
     memset(pTag, 0, sizeof(STagData));
+    pCmd->payloadLen = sizeof(STagData);
     
     /*
      * the source super table is moved to the secondary position of the pTableMetaInfo list

src/connector/python/linux/python3/taos/cursor.py

 from .cinterface import CTaosInterface
 from .error import *
 
+# querySeqNum = 0
+
 class TDengineCursor(object):
     """Database cursor which is used to manage the context of a fetch operation.
 
@@ -109,7 +111,14 @@ class TDengineCursor(object):
         if params is not None:
             pass
         
+
+        # global querySeqNum
+        # querySeqNum += 1
+        # localSeqNum = querySeqNum # avoid raice condition
+        # print("   >> Exec Query ({}): {}".format(localSeqNum, str(stmt)))
         res = CTaosInterface.query(self._connection._conn, stmt)
+        # print("   << Query ({}) Exec Done".format(localSeqNum))
+
         if res == 0:
             if CTaosInterface.fieldsCount(self._connection._conn) == 0:
                 self._affected_rows += CTaosInterface.affectedRows(self._connection._conn)

src/inc/taosdef.h

@@ -252,12 +252,12 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MULTI_METERMETA_MAX_NUM    100000  // maximum batch size allowed to load metermeta
 
 #define TSDB_MIN_CACHE_BLOCK_SIZE       1
-#define TSDB_MAX_CACHE_BLOCK_SIZE       10240   // 10GB for each vnode
+#define TSDB_MAX_CACHE_BLOCK_SIZE       128     // 128MB for each vnode
 #define TSDB_DEFAULT_CACHE_BLOCK_SIZE   16
 
 #define TSDB_MIN_TOTAL_BLOCKS           2
 #define TSDB_MAX_TOTAL_BLOCKS           10000
-#define TSDB_DEFAULT_TOTAL_BLOCKS       2
+#define TSDB_DEFAULT_TOTAL_BLOCKS       4
 
 #define TSDB_MIN_TABLES                 4
 #define TSDB_MAX_TABLES                 200000

src/mnode/src/mgmtTable.c

@@ -1252,7 +1252,7 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
   pMeta->contLen = htons(pMeta->contLen);
   rpcSendResponse(&rpcRsp);
 
-  mTrace("stable:%%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
+  mTrace("stable:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
 }
 
 static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
@@ -1759,7 +1759,7 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
   newMsg->msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
   newMsg->pCont = pCreateMsg;
 
-  mTrace("table:%s, start to create on demand", pInfo->tableId);
+  mTrace("table:%s, start to create on demand, stable:%s", pInfo->tableId, pInfo->tags);
   mgmtAddToShellQueue(newMsg);
 }
 

tests/pytest/crash_gen.sh

+#!/bin/bash
+
+# This is the script for us to try to cause the TDengine server or client to crash
+#    
+# PREPARATION
+#
+# 1. Build an compile the TDengine source code that comes with this script, in the same directory tree
+# 2. Please follow the direction in our README.md, and build TDengine in the build/ directory
+# 3. Adjust the configuration file if needed under build/test/cfg/taos.cfg
+# 4. Run the TDengine server instance: cd build; ./build/bin/taosd -c test/cfg
+# 5. Make sure you have a working Python3 environment: run /usr/bin/python3 --version, and you should get 3.6 or above
+# 6. Make sure you have the proper Python packages: # sudo apt install python3-setuptools python3-pip python3-distutils
+#
+# RUNNING THIS SCRIPT
+# 
+# This script assumes the source code directory is intact, and that the binaries has been built in the
+# build/ directory, as such, will will load the Python libraries in the directory tree, and also load
+# the TDengine client shared library (so) file, in the build/directory, as evidenced in the env
+# variables below.
+#
+# Running the script is simple, no parameter is needed (for now, but will change in the future).
+#
+# Happy Crashing...
+
+
+# Due to the heavy path name assumptions/usage, let us require that the user be in the current directory
+EXEC_DIR=`dirname "$0"`
+if [[ $EXEC_DIR != "." ]]
+then
+    echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)"
+    exit -1
+fi
+
+# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work.
+export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3
+
+# Then let us set up the library path so that our compiled SO file can be loaded by Python
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib
+
+# Now we are all let, and let's see if we can find a crash. Note we pass all params
+./crash_gen.py $@

tests/pytest/crash_gen_0519.sh

+#!/bin/bash
+
+# This is the script for us to try to cause the TDengine server or client to crash
+#    
+# PREPARATION
+#
+# 1. Build an compile the TDengine source code that comes with this script, in the same directory tree
+# 2. Please follow the direction in our README.md, and build TDengine in the build/ directory
+# 3. Adjust the configuration file if needed under build/test/cfg/taos.cfg
+# 4. Run the TDengine server instance: cd build; ./build/bin/taosd -c test/cfg
+# 5. Make sure you have a working Python3 environment: run /usr/bin/python3 --version, and you should get 3.6 or above
+# 6. Make sure you have the proper Python packages: # sudo apt install python3-setuptools python3-pip python3-distutils
+#
+# RUNNING THIS SCRIPT
+# 
+# This script assumes the source code directory is intact, and that the binaries has been built in the
+# build/ directory, as such, will will load the Python libraries in the directory tree, and also load
+# the TDengine client shared library (so) file, in the build/directory, as evidenced in the env
+# variables below.
+#
+# Running the script is simple, no parameter is needed (for now, but will change in the future).
+#
+# Happy Crashing...
+
+
+# Due to the heavy path name assumptions/usage, let us require that the user be in the current directory
+EXEC_DIR=`dirname "$0"`
+if [[ $EXEC_DIR != "." ]]
+then
+    echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)"
+    exit -1
+fi
+
+# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work.
+export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3
+
+# Then let us set up the library path so that our compiled SO file can be loaded by Python
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib
+
+# Now we are all let, and let's see if we can find a crash. Note we pass all params
+./crash_gen_0519.py $@

tests/script/general/http/autocreate.sim

+system sh/stop_dnodes.sh
+sleep 3000
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c wallevel -v 0
+system sh/cfg.sh -n dnode1 -c http -v 1
+system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1
+system sh/exec.sh -n dnode1 -s start
+
+sleep 3000
+sql connect
+
+print ============================ dnode1 start
+
+print ===============  step1 - prepare data
+sql create database db
+sql use db
+sql create table if not exists db.win_cpu(ts timestamp,f_percent_dpc_time double,f_percent_idle_time double,f_percent_interrupt_time double,f_percent_privileged_time double,f_percent_processor_time double,f_percent_user_time double) tags(t_host binary(32),t_instance binary(32),t_objectname binary(32));
+
+print ===============  step2 - auto create
+
+system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'import into db.win_cpu_windows_1_processor using db.win_cpu tags('windows','1','Processor') values(1564641722000,0.000000,95.598305,0.000000,0.000000,0.000000,0.000000);' 127.0.0.1:6020/rest/sql
+print curl 127.0.0.1:6020/rest/sql -----> $system_content
+#if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then
+#  return -1
+#endi
+
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

tests/script/general/table/column_value.sim

@@ -60,11 +60,9 @@ if $rows != 0 then
 endi
 
 print =============== step4
-sql create table  $tb (ts timestamp, speed double, v1 binary(1500), v2 binary(1500), v3 binary(1500), v4 binary(500), v5 binary(500)) -x step4
-	return -1
-step4:
+sql create table  $tb (ts timestamp, speed double, v1 binary(1500), v2 binary(1500), v3 binary(1500), v4 binary(500), v5 binary(500)) 
 sql show tables
-if $rows != 0 then 
+if $rows != 1 then 
   return -1
 endi
 

tests/script/general/tag/create.sim

@@ -579,7 +579,7 @@ $i = 30
 $mt = $mtPrefix . $i
 $tb = $tbPrefix . $i
 sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol binary(250), tgcol2 binary(250), tgcol3 binary(30)) -x step30
-  return -1
+#  return -1
 step30: 
 
 print =============== step31

tests/script/jenkins/basic.txt

@@ -53,7 +53,7 @@ cd ../../../debug; make
 ./test.sh -f general/db/delete_reuse1.sim
 ./test.sh -f general/db/delete_reuse2.sim
 ./test.sh -f general/db/delete_reusevnode.sim
-./test.sh -f general/db/delete_reusevnode2.sim
+#hongze ./test.sh -f general/db/delete_reusevnode2.sim
 ./test.sh -f general/db/delete_writing1.sim
 ./test.sh -f general/db/delete_writing2.sim
 ./test.sh -f general/db/len.sim
@@ -250,7 +250,7 @@ cd ../../../debug; make
 ./test.sh -u -f unique/cluster/balance3.sim
 ./test.sh -u -f unique/cluster/cache.sim
 
-./test.sh -u -f unique/column/replica3.sim
+#jeff ./test.sh -u -f unique/column/replica3.sim
 
 #liao wait ./test.sh -u -f unique/db/commit.sim
 ./test.sh -u -f unique/db/delete.sim
@@ -302,4 +302,4 @@ cd ../../../debug; make
 ./test.sh -u -f unique/vnode/replica2_repeat.sim
 ./test.sh -u -f unique/vnode/replica3_basic.sim
 ./test.sh -u -f unique/vnode/replica3_repeat.sim
-./test.sh -u -f unique/vnode/replica3_vgroup.sim
+#jeff ./test.sh -u -f unique/vnode/replica3_vgroup.sim