Merge branch 'develop' into test/testcase

Jenkinsfile

@@ -86,7 +86,7 @@ pipeline {
             pre_test()
             sh '''
             cd ${WKC}/tests
-            ./test-all.sh pytest
+            ./test-all.sh pytestfq
             date'''
           }
         }
@@ -96,7 +96,7 @@ pipeline {
             pre_test()
             sh '''
             cd ${WKC}/tests
-            ./test-all.sh b1
+            ./test-all.sh b1fq
             date'''
           }
         }
@@ -120,7 +120,7 @@ pipeline {
             sh '''
             date
             cd ${WKC}/tests
-            ./test-all.sh b2
+            ./test-all.sh b2fq
             date
             '''
           }
@@ -141,7 +141,7 @@ pipeline {
             sh '''
             date
             cd ${WKC}/tests
-            ./test-all.sh b3
+            ./test-all.sh b3fq
             date'''
           }
         }

documentation20/webdocs/markdowndocs/connector-java-ch.md

 #  Java Connector
 
-Java连接器支持的系统有： Linux 64/Windows x64/Windows x86。
+Java连接器支持的系统有：
+| **CPU类型**  | x64（64bit） |          |          | ARM64  | ARM32  |
+| ------------ | ------------ | -------- | -------- | -------- | -------- |
+| **OS类型**   | Linux        | Win64    | Win32    | Linux    | Linux    |
+| **支持与否** | **支持**     | **支持** | **支持** | **支持** | **支持** |
 
 TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。
 
 由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。
 
-* libtaos.so 
+* libtaos.so
     在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。
-    
+
 * taos.dll
     在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。
-    
+
 > 注意：在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。
 
 TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致 taos-jdbcdriver 并未完全实现 JDBC 标准规范。在使用时需要注意以下几点：
@@ -21,7 +25,6 @@ TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一
 * 目前不支持表间的 union 操作。
 * 目前不支持嵌套查询(nested query)，对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver 则会自动关闭上一个 ResultSet。
 
-
 ## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本
 
 | taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 |
@@ -70,7 +73,6 @@ maven 项目中使用如下 pom.xml 配置即可：
 
 下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。
 
-
 ## 使用说明
 
 ### 获取连接
@@ -212,7 +214,6 @@ while(resultSet.next()){
 ```
 > 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。
 
-
 ### 订阅
 
 #### 创建
@@ -227,7 +228,7 @@ TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from met
 * sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据
 * restart：如果订阅已经存在，是重新开始，还是继续之前的订阅
 
-如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic' 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。
+如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。
 
 #### 消费数据
 
@@ -255,7 +256,6 @@ sub.close(true);
 
 `close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。
 
-
 ### 关闭资源
 
 ```java
@@ -263,7 +263,7 @@ resultSet.close();
 stmt.close();
 conn.close();
 ```
-> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。  
+> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。
 
 ## 与连接池使用
 
@@ -289,18 +289,18 @@ conn.close();
     config.setMinimumIdle(3);           //minimum number of idle connection
     config.setMaximumPoolSize(10);      //maximum number of connection in the pool
     config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool
-    config.setIdleTimeout(60000);       // max idle time for recycle idle connection 
+    config.setIdleTimeout(60000);       // max idle time for recycle idle connection
     config.setConnectionTestQuery("describe log.dn"); //validation query
     config.setValidationTimeout(3000);   //validation query timeout
 
     HikariDataSource ds = new HikariDataSource(config); //create datasource
-    
+
     Connection  connection = ds.getConnection(); // get connection
     Statement statement = connection.createStatement(); // get statement
-    
-    //query or insert 
+
+    //query or insert
     // ...
-    
+
     connection.close(); // put back to conneciton pool
 }
 ```
@@ -342,7 +342,7 @@ public static void main(String[] args) throws Exception {
     properties.put("testWhileIdle","true"); // test connection while idle
     properties.put("testOnBorrow","false"); // don't need while testWhileIdle is true
     properties.put("testOnReturn","false"); // don't need while testWhileIdle is true
-    
+
     //create druid datasource
     DataSource ds = DruidDataSourceFactory.createDataSource(properties);
     Connection  connection = ds.getConnection(); // get connection
@@ -376,15 +376,15 @@ Query OK, 1 row(s) in set (0.000141s)
 ## 常见问题
 
 * java.lang.UnsatisfiedLinkError: no taos in java.library.path
-  
+
   **原因**：程序没有找到依赖的本地函数库 taos。
-  
+
   **解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 ` ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。
-  
+
 * java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform
-  
+
   **原因**：目前 TDengine 只支持 64 位 JDK。
-  
+
   **解决方法**：重新安装 64 位 JDK。
 
 * 其它问题请参考 [Issues][7]
@@ -399,7 +399,7 @@ Query OK, 1 row(s) in set (0.000141s)
 [8]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver
 [9]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver
 [10]: https://maven.aliyun.com/mvn/search
-[11]:  https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate
+[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate
 [12]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo
 [13]: https://www.taosdata.com/cn/documentation20/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE
 [14]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client

src/util/src/tskiplist.c

@@ -280,7 +280,10 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
   tSkipListRLock(pSkipList);
 
   if (iter->order == TSDB_ORDER_ASC) {
-    if (iter->cur == pSkipList->pTail) return false;
+    if (iter->cur == pSkipList->pTail) {
+      tSkipListUnlock(pSkipList);
+      return false;
+    }
     iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
 
     // a new node is inserted into between iter->cur and iter->next, ignore it

tests/pytest/concurrent_inquiry.py

@@ -38,7 +38,7 @@ class ConcurrentInquiry:
     #             stableNum = 2,subtableNum = 1000,insertRows = 100):  
     def __init__(self,ts,host,user,password,dbname,
                 stb_prefix,subtb_prefix,n_Therads,r_Therads,probabilities,loop,
-                stableNum ,subtableNum ,insertRows ):  
+                stableNum ,subtableNum ,insertRows ,mix_table):  
         self.n_numOfTherads = n_Therads
         self.r_numOfTherads = r_Therads
         self.ts=ts
@@ -60,6 +60,7 @@ class ConcurrentInquiry:
         self.stableNum = stableNum
         self.subtableNum = subtableNum
         self.insertRows = insertRows
+        self.mix_table = mix_table
     def SetThreadsNum(self,num):
         self.numOfTherads=num
 
@@ -195,7 +196,13 @@ class ConcurrentInquiry:
                 pick_func+=alias
             sel_col_list.append(pick_func)
             
-        sql=sql+','.join(sel_col_list)+' from '+random.choice(self.stb_list+self.subtb_list)+' '                        #select col & func
+        sql=sql+','.join(sel_col_list)         #select col & func
+        if self.mix_table == 0:
+            sql = sql + ' from '+random.choice(self.stb_list+self.subtb_list)+' '         
+        elif self.mix_table == 1:
+            sql = sql + ' from '+random.choice(self.subtb_list)+' '
+        else:
+            sql = sql + ' from '+random.choice(self.stb_list)+' ' 
         con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill]
         sel_con=random.sample(con_func,random.randint(0,len(con_func)))
         sel_con_list=[]
@@ -212,8 +219,23 @@ class ConcurrentInquiry:
         col_intersection = []
         tag_intersection = []
         subtable = None
-
-        if bool(random.getrandbits(1)):
+        if self.mix_table == 0:
+            if bool(random.getrandbits(1)):
+                subtable = True
+                tbname = random.sample(self.subtb_list,2)
+                for i in tbname:
+                    col_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
+                    tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
+                col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
+                tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
+            else:
+                tbname = random.sample(self.stb_list,2)
+                for i in tbname:
+                    col_list.append(self.stb_stru_list[self.stb_list.index(i)])
+                    tag_list.append(self.stb_stru_list[self.stb_list.index(i)])
+                col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
+                tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
+        elif self.mix_table == 1:
             subtable = True
             tbname = random.sample(self.subtb_list,2)
             for i in tbname:
@@ -228,12 +250,9 @@ class ConcurrentInquiry:
                 tag_list.append(self.stb_stru_list[self.stb_list.index(i)])
             col_intersection = list(set(col_list[0]).intersection(set(col_list[1])))
             tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1])))
-        
-        
         con_rand=random.randint(0,len(condition_list))
         col_rand=random.randint(0,len(col_list))
         tag_rand=random.randint(0,len(tag_list))
-        
         sql='select '                                           #select 
         
         sel_col_tag=[]
@@ -247,12 +266,16 @@ class ConcurrentInquiry:
 
         sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 '                        #select col & func
         join_section = None
+        temp = None
         if subtable:
-            join_section = ''.join(random.choices(col_intersection))
-            sql += 'where t1._c0 = t2._c0 and ' + 't1.' + join_section + '=t2.' + join_section
+            temp = random.choices(col_intersection)
+            join_section = temp.pop()
+            sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section)
         else:
-            join_section = ''.join(random.choices(col_intersection+tag_intersection))
-            sql += 'where t1._c0 = t2._c0 and ' + 't1.' + join_section + '=t2.' + join_section
+            temp = random.choices(col_intersection+tag_intersection)
+            join_section = temp.pop()
+            print(random.choices(col_intersection))
+            sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section)
         return sql
 
     def random_pick(self): 
@@ -516,12 +539,20 @@ parser.add_argument(
     default=2,
     type=int,
     help='Number of stables  (default: 2)')
+parser.add_argument(
+    '-m',
+    '--mix-stable-subtable',
+    action='store',
+    default=0,
+    type=int,
+    help='0:stable & substable ,1:subtable ,2:stable (default: 0)')
 
 args = parser.parse_args()
 q = ConcurrentInquiry(
     args.ts,args.host_name,args.user,args.password,args.db_name,
                 args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads,
-                args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records )
+                args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records,
+                args.mix_stable_subtable )
 
 if args.create_table: 
     q.gen_data()

tests/test-all.sh

@@ -25,6 +25,24 @@ function runSimCaseOneByOne {
     fi
   done < $1
 }
+function runSimCaseOneByOnefq {
+  while read -r line; do
+    if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then
+			case=`echo $line | grep sim$ |awk '{print $NF}'`
+
+      start_time=`date +%s`
+      ./test.sh -f $case > /dev/null 2>&1 && \
+        echo -e "${GREEN}$case success${NC}" | tee -a out.log || \
+        echo -e "${RED}$case failed${NC}" | tee -a out.log
+      out_log=`tail -1 out.log  `
+      if [[ $out_log =~ 'failed' ]];then
+        exit 8
+      fi
+      end_time=`date +%s`
+      echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log
+    fi
+  done < $1
+}
 
 function runPyCaseOneByOne {
   while read -r line; do
@@ -52,7 +70,32 @@ function runPyCaseOneByOne {
     fi
   done < $1
 }
-
+function runPyCaseOneByOnefq {
+  while read -r line; do
+    if [[ $line =~ ^python.* ]]; then
+      if [[ $line != *sleep* ]]; then
+        
+        if [[ $line =~ '-r' ]];then
+          case=`echo $line|awk '{print $4}'`
+        else
+          case=`echo $line|awk '{print $NF}'`
+        fi
+        start_time=`date +%s`
+        $line > /dev/null 2>&1 && \
+          echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \
+          echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log
+        end_time=`date +%s`
+        out_log=`tail -1 pytest-out.log  `
+        if [[ $out_log =~ 'failed' ]];then
+          exit 8
+        fi
+        echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
+      else
+        $line > /dev/null 2>&1
+      fi
+    fi
+  done < $1
+}
 totalFailed=0
 totalPyFailed=0
 
@@ -78,6 +121,15 @@ if [ "$2" != "python" ]; then
   elif [ "$1" == "b3" ]; then
     echo "### run TSIM b3 test ###"
     runSimCaseOneByOne jenkins/basic_3.txt
+  elif [ "$1" == "b1fq" ]; then
+    echo "### run TSIM b1 test ###"
+    runSimCaseOneByOnefq jenkins/basic_1.txt
+  elif [ "$1" == "b2fq" ]; then
+    echo "### run TSIM b2 test ###"
+    runSimCaseOneByOnefq jenkins/basic_2.txt
+  elif [ "$1" == "b3fq" ]; then
+    echo "### run TSIM b3 test ###"
+    runSimCaseOneByOnefq jenkins/basic_3.txt
   elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
     echo "### run TSIM smoke test ###"
     runSimCaseOneByOne basicSuite.sim
@@ -137,6 +189,9 @@ if [ "$2" != "sim" ]; then
   elif [ "$1" == "pytest" ]; then
     echo "### run Python full test ###"
     runPyCaseOneByOne fulltest.sh
+  elif [ "$1" == "pytestfq" ]; then
+    echo "### run Python full test ###"
+    runPyCaseOneByOnefq fulltest.sh
   elif [ "$1" == "p1" ]; then
     echo "### run Python_1 test ###"
     runPyCaseOneByOne pytest_1.sh