Merge branch 'develop' into cpwu/TD-11256

documentation20/cn/12.taos-sql/docs.md

@@ -53,6 +53,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传
 | 8    |  TINYINT  | 1      | 单字节整型，范围 [-127, 127], -128 用于 NULL                                |
 | 9    |   BOOL    | 1      | 布尔型，{true, false}                                      |
 | 10   |   NCHAR   | 自定义 | 记录包含多字节字符在内的字符串，如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小，类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符，会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度，将会报错。 |
+| 11   |   JSON    |       | json数据类型， 只有tag类型可以是json格式                                    |
 <!-- REPLACE_OPEN_TO_ENTERPRISE__COLUMN_TYPE_ADDONS -->
 
 **Tips**:
@@ -1603,6 +1604,15 @@ TAOS SQL 支持对标签、TBNAME 进行 GROUP BY 操作，也支持普通列进
 
 IS NOT NULL 支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。
 
+**ORDER BY的限制**
+
+- 非超级表只能有一个order by.
+- 超级表最多两个order by， 并且第二个必须为ts.
+- order by tag，必须和group by tag一起，并且是同一个tag。 tbname和tag一样逻辑。  只适用于超级表
+- order by 普通列，必须和group by一起或者和top/bottom一起，并且是同一个普通列。 适用于超级表和普通表。如果同时存在 group by和 top/bottom一起，order by优先必须和group by同一列。
+- order by ts. 适用于超级表和普通表。
+- order by ts同时含有group by时 针对group内部用ts排序
+
 ## 表(列)名合法性说明
 TDengine 中的表（列）名命名规则如下：
 只能由字母、数字、下划线构成，数字不能在首位，长度不能超过192字节，不区分大小写。
@@ -1618,3 +1628,87 @@ TDengine 中的表（列）名命名规则如下：
 
 支持版本
 支持转义符的功能从 2.3.0.1 版本开始。
+
+
+## Json类型使用说明
+- 语法说明
+
+  1. 创建json类型tag
+
+     ```mysql
+     create stable s1 (ts timestamp, v1 int) tags (info json)
+
+     create table s1_1 using s1 tags ('{"k1": "v1"}')
+     ```
+  3. json取值操作符 ->
+
+     ```mysql   
+     select * from s1 where info->'k1' = 'v1'
+
+     select info->'k1' from s1 
+     ```
+  4. json key是否存在操作符 contains
+
+     ```mysql
+     select * from s1 where info contains 'k2'
+    
+     select * from s1 where info contains 'k1'
+     ```
+     
+- 支持的操作
+
+  1. 在where条件中时，支持函数match/nmatch/between and/like/and/or/is null/is no null，不支持in
+
+     ```mysql 
+     select * from s1 where info→'k1' match 'v*'; 
+
+     select * from s1 where info→'k1' like 'v%' and info contains 'k2';
+
+     select * from s1 where info is null; 
+  
+     select * from s1 where info->'k1' is not null
+     ```
+
+  2. 支持json tag放在group by、order by、join子句、union all以及子查询中，比如group by json->'key'
+
+  3. 支持distinct操作.
+
+     ```mysql 
+     select distinct info→'k1' from s1
+     ```
+  
+  5. 标签操作
+    
+     支持修改json标签值（全量覆盖）
+
+     支持修改json标签名 
+  
+     不支持添加json标签、删除json标签、修改json标签列宽
+
+- 其他约束条件
+  
+  1. 只有标签列可以使用json类型，如果用json标签，标签列只能有一个。
+  
+  2. 长度限制：json 中key的长度不能超过256，并且key必须为可打印ascii字符；json字符串总长度不超过4096个字节。
+
+  3. json格式限制：
+
+     1. json输入字符串可以为空（"","\t"," "或null）或object，不能为非空的字符串，布尔型和数组。
+     2. object 可为{}，如果object为{}，则整个json串记为空。key可为""，若key为""，则json串中忽略该k-v对。
+     3. value可以为数字(int/double)或字符串或bool或null，暂不可以为数组。不允许嵌套。
+     4. 若json字符串中出现两个相同的key，则第一个生效。
+     5. json字符串里暂不支持转义。
+  
+  4. 当查询json中不存在的key时，返回NULL
+
+  5. 当json tag作为子查询结果时，不再支持上层查询继续对子查询中的json串做解析查询。
+  
+     比如暂不支持 
+     ```mysql 
+     select jtag→'key' from (select jtag from stable)
+     ```
+  
+     不支持 
+     ```mysql 
+     select jtag->'key' from (select jtag from stable) where jtag->'key'>0
+     ```

documentation20/en/12.taos-sql/docs.md

@@ -53,7 +53,7 @@ In TDengine, the following 10 data types can be used in data model of an ordinar
 | 8    | TINYINT       | 1         | A nullable integer type with a range of [-127, 127]          |
 | 9    | BOOL          | 1         | Boolean type，{true, false}                                  |
 | 10   | NCHAR         | Custom    | Used to record non-ASCII strings, such as Chinese characters. Each nchar character takes up 4 bytes of storage space. Single quotation marks are used at both ends of the string, and escape characters are required for single quotation marks in the string, that is \’. When nchar is used, the string size must be specified. A column of type nchar (10) indicates that the string of this column stores up to 10 nchar characters, which will take up 40 bytes of space. If the length of the user string exceeds the declared length, an error will be reported. |
-
+| 11   | JSON          |           | Json type，only support for tag                                  |
 
 
 **Tips**:
@@ -1245,3 +1245,92 @@ TAOS SQL supports join columns of two tables by Primary Key timestamp between th
 **Availability of is no null**
 
 Is not null supports all types of columns. Non-null expression is < > "" and only applies to columns of non-numeric types.
+
+**Restrictions on order by**
+
+- A non super table can only have one order by.
+- The super table can have at most two order by expression, and the second must be ts.
+- Order by tag must be the same tag as group by tag. TBNAME is as logical as tag.
+- Order by ordinary column must be the same ordinary column as group by or top/bottom. If both group by and top / bottom exist, order by must be in the same column as group by.
+- There are both order by and group by. The internal of the group is sorted by ts
+- Order by ts.
+
+## JSON type instructions
+- Syntax description
+
+  1. Create JSON type tag
+
+     ```mysql
+     create stable s1 (ts timestamp, v1 int) tags (info json)
+
+     create table s1_1 using s1 tags ('{"k1": "v1"}')
+     ```
+  3. JSON value operator(->)
+
+     ```mysql   
+     select * from s1 where info->'k1' = 'v1'
+
+     select info->'k1' from s1 
+     ```
+  4. JSON key existence operator(contains)
+
+     ```mysql
+     select * from s1 where info contains 'k2'
+    
+     select * from s1 where info contains 'k1'
+     ```
+
+- Supported operations
+
+  1. In where condition，support match/nmatch/between and/like/and/or/is null/is no null，in operator is not support.
+
+     ```mysql 
+     select * from s1 where info→'k1' match 'v*'; 
+
+     select * from s1 where info→'k1' like 'v%' and info contains 'k2';
+
+     select * from s1 where info is null; 
+  
+     select * from s1 where info->'k1' is not null
+     ```
+
+  2. JSON tag is supported in group by、order by、join clause、union all and subquery，like group by json->'key'
+
+  3. Support distinct operator.
+
+     ```mysql 
+     select distinct info→'k1' from s1
+     ```
+
+  5. Tag
+
+     Support change JSON tag（full coverage）
+
+     Support change the name of JSON tag
+
+     Not support add JSON tag, delete JSON tag
+
+- Other constraints
+
+  1. Only tag columns can use JSON type. If JSON tag is used, there can only be one tag column.
+
+  2. Length limit:The length of the key in JSON cannot exceed 256, and the key must be printable ASCII characters; The total length of JSON string does not exceed 4096 bytes.
+
+  3. JSON format restrictions:
+
+    1. JSON input string can be empty (""," ","\t" or null) or object, and cannot be nonempty string, boolean or array.
+    2. Object can be {}, if the object is {}, the whole JSON string is marked as empty. The key can be "", if the key is "", the K-V pair will be ignored in the JSON string.
+    3. Value can be a number (int/double) or string, bool or null, not an array. Nesting is not allowed.
+    4. If two identical keys appear in the JSON string, the first one will take effect.
+    5. Escape is not supported in JSON string.
+
+  4. Null is returned when querying the key that does not exist in JSON.
+
+  5. When JSON tag is used as the sub query result, parsing and querying the JSON string in the sub query is no longer supported in the upper level query.
+
+     The following query is not supported:
+     ```mysql 
+     select jtag→'key' from (select jtag from stable)
+      
+     select jtag->'key' from (select jtag from stable) where jtag->'key'>0
+     ```

packaging/release.sh

@@ -494,11 +494,13 @@ else
   exit 1
 fi
 
+CORES=`grep -c ^processor /proc/cpuinfo`
+
 if [[ "$allocator" == "jemalloc" ]]; then
     # jemalloc need compile first, so disable parallel build
-    make -j 8 && ${csudo}make install
+    make -j ${CORES} && ${csudo}make install
 else
-    make -j 8 && ${csudo}make install
+    make -j ${CORES} && ${csudo}make install
 fi
 
 cd ${curr_dir}

src/balance/CMakeLists.txt

@@ -9,3 +9,8 @@ INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 ADD_LIBRARY(balance ${SRC})
+
+IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
+    ADD_DEPENDENCIES(balance jemalloc)
+ENDIF ()
+

src/kit/shell/CMakeLists.txt

@@ -32,6 +32,10 @@ ELSEIF (TD_WINDOWS)
   LIST(APPEND SRC ./src/shellMain.c)
   LIST(APPEND SRC ./src/shellWindows.c)
   ADD_EXECUTABLE(shell ${SRC})
+  IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
+    ADD_DEPENDENCIES(shell jemalloc)
+  ENDIF ()
+
   TARGET_LINK_LIBRARIES(shell taos_static cJson)
 
   IF (TD_POWER)

taos-tools @ a3611888

-Subproject commit beca4813316f254624d8dbecf54d45a5a232c61d
+Subproject commit a3611888d4257a9baa0ce876b04b47c60cc17279

src/mnode/CMakeLists.txt

@@ -8,3 +8,8 @@ INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 ADD_LIBRARY(mnode ${SRC})
+
+IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
+    ADD_DEPENDENCIES(mnode jemalloc)
+ENDIF ()
+

src/os/src/detail/CMakeLists.txt

@@ -11,6 +11,11 @@ ADD_LIBRARY(os ${SRC})
 
 IF (TD_LINUX)
   TARGET_LINK_LIBRARIES(os oslinux)
+
+  IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
+    ADD_DEPENDENCIES(os jemalloc)
+  ENDIF ()
+
   IF (TD_ARM_32 OR TD_LINUX_32)
     TARGET_LINK_LIBRARIES(os atomic)
   ENDIF ()

src/os/src/linux/CMakeLists.txt

@@ -5,3 +5,8 @@ AUX_SOURCE_DIRECTORY(. SRC)
 ADD_LIBRARY(oslinux ${SRC})
 
 TARGET_LINK_LIBRARIES(oslinux m rt z dl)
+
+IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
+    ADD_DEPENDENCIES(oslinux jemalloc)
+ENDIF ()
+

taosadapter @ 11d1e022

-Subproject commit 826f3d3b7820a5c007d301854d56db003b424d0a
+Subproject commit 11d1e02255edfeeaa8d5b1f45abfa9637332ce65

src/query/src/qAggMain.c

@@ -4870,7 +4870,8 @@ static void elapsedFinalizer(SQLFunctionCtx *pCtx) {
   }
 
   SElapsedInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
-  *(double *)pCtx->pOutput = (double)pInfo->max - (double)pInfo->min;
+  double result = (double)pInfo->max - (double)pInfo->min;
+  *(double *)pCtx->pOutput = result >= 0 ? result : -result;
   if (pCtx->numOfParams > 0 && pCtx->param[0].i64 > 0) {
     *(double *)pCtx->pOutput = *(double *)pCtx->pOutput / pCtx->param[0].i64;
   }

tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import subprocess
+
+
+class TDTestCase:
+    def caseDescription(self):
+        '''
+        case1<sdsang>: [TD-12362] taosdump supports JSON
+        '''
+        return
+
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        self.tmpdir = "tmp"
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosdump" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute("drop database if exists db")
+        tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")
+
+        tdSql.execute("use db")
+        tdSql.execute(
+            "create table st(ts timestamp, c1 int) tags(jtag JSON)")
+        tdSql.execute("create table t1 using st tags('{\"location\": \"beijing\"}')")
+        tdSql.execute("insert into t1 values(1500000000000, 1)")
+
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosdump not found!")
+        else:
+            tdLog.info("taosdump found in %s" % buildPath)
+        binPath = buildPath + "/build/bin/"
+
+        if not os.path.exists(self.tmpdir):
+            os.makedirs(self.tmpdir)
+        else:
+            print("directory exists")
+            os.system("rm -rf %s" % self.tmpdir)
+            os.makedirs(self.tmpdir)
+
+        os.system("%staosdump --databases db -o %s" % (binPath, self.tmpdir))
+
+        tdSql.execute("drop database db")
+
+        os.system("%staosdump -i %s" % (binPath, self.tmpdir))
+
+        tdSql.query("show databases")
+        tdSql.checkRows(1)
+
+        tdSql.execute("use db")
+        tdSql.query("show stables")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'st')
+
+        tdSql.query("show tables")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 't1')
+
+        tdSql.query("select jtag->'location' from st")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, "\"beijing\"")
+
+        tdSql.query("select * from st where jtag contains 'location'")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(0, 2, '{\"location\":\"beijing\"}')
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/develop-test/fulltest-tools.sh

+python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeJson.py

tests/pytest/functions/function_elapsed_case.py

@@ -222,6 +222,13 @@ class ElapsedCase:
         self.orderbyForStableCheck("select elapsed(ts) from st1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00' interval(222m) group by tbname", 1, 2)
         self.orderbyForStableCheck("select elapsed(ts) from st1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00' interval(1000m) group by tbname", 1, 2)
 
+        #nested query
+        resAsc = tdSql.getResult("select elapsed(ts) from (select csum(i) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00')")
+        resDesc = tdSql.getResult("select elapsed(ts) from (select csum(i) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00' order by ts desc)")
+        resRows = len(resAsc)
+        for i in range(resRows):
+            tdSql.checkEqual(resAsc[i][0], resDesc[resRows - i - 1][0])
+
     def slimitCheck(self, sql):
         tdSql.checkEqual(tdSql.query(sql + " slimit 0"), 0)
         tdSql.checkEqual(tdSql.query(sql + " slimit 1 soffset 0"), tdSql.query(sql + " slimit 0, 1"))

tests/system-test/fulltest-insert.sh

-python3 ./test.py -f 1-insert/batchInsert.py
+#python3 ./test.py -f 1-insert/batchInsert.py

tests/system-test/fulltest-others.sh

-python3 ./test.py -f 0-others/json_tag.py
+python3 ./test.py -f 0-others/create_col_tag.py

tests/system-test/fulltest-query.sh

+
 python3 ./test.py -f 2-query/ts_hidden_column.py
 python3 ./test.py -f 2-query/union-order.py
 python3 test.py -f 2-query/TD-11256.py
+
+python3 ./test.py -f 2-query/TD-11389.py
+python3 ./test.py -f 2-query/TD-11945_crash.py
+python3 ./test.py -f 2-query/TD-12340-12342.py
+python3 ./test.py -f 2-query/TD-12344.py

tests/system-test/fulltest-taosAdapter.sh

+python3 ./test.py -f 4-taosAdapter/taosAdapter_insert.py
+python3 ./test.py -f 4-taosAdapter/taosAdapter_query.py

tests/system-test/fulltest-tools.sh

+python3 ./test.py -f 5-taos-tools/basic.py