Merge branch 'develop' into feature/TD-2581

documentation20/cn/08.connector/01.java/docs.md

@@ -49,6 +49,7 @@ TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致
 </tr>
 </table>
 
+注意：与 JNI 方式不同，RESTful 接口是无状态的，因此 `USE db_name` 指令没有效果，RESTful 下所有对表名、超级表名的引用都需要指定数据库名前缀。
 
 ## 如何获取 taos-jdbcdriver
 
@@ -551,7 +552,7 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对
 | BIGINT            | java.lang.Long     |
 | FLOAT             | java.lang.Float    |
 | DOUBLE            | java.lang.Double   |
-| SMALLINT	    | java.lang.Short    |
+| SMALLINT          | java.lang.Short    |
 | TINYINT           | java.lang.Byte     |
 | BOOL              | java.lang.Boolean  |
 | BINARY            | byte array         |

documentation20/cn/08.connector/docs.md

@@ -585,7 +585,9 @@ conn.close()
 
 ## <a class="anchor" id="restful"></a>RESTful Connector
 
-为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。RESTful连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1965.html)。
+为支持各种不同类型平台的开发，TDengine 提供符合 REST 设计标准的 API，即 RESTful API。为最大程度降低学习成本，不同于其他数据库 RESTful API 的设计方法，TDengine 直接通过 HTTP POST 请求 BODY 中包含的 SQL 语句来操作数据库，仅需要一个 URL。RESTful 连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1965.html)。
+
+注意：与标准连接器的一个区别是，RESTful 接口是无状态的，因此 `USE db_name` 指令没有效果，所有对表名、超级表名的引用都需要指定数据库名前缀。
 
 ### HTTP请求格式 
 

documentation20/cn/12.taos-sql/docs.md

@@ -144,7 +144,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传
     ```
     UPDATE 参数控制是否允许更新数据。缺省值为 0，取值范围为 [0, 1]。0 表示会直接丢弃后写入的相同时间戳的数据；1 表示会使用后写入的数据覆盖已有的相同时间戳的数据。
 
-    **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。另外，从 2.1.1.0 版本开始，修改这些参数后无需重启服务器即可生效。
+    **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。另外，从 2.1.3.0 版本开始，修改这些参数后无需重启服务器即可生效。
 
 - **显示系统所有数据库**
 
@@ -263,6 +263,14 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传
     ```
     如果表是通过超级表创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构。
 
+- **表修改列宽**
+
+    ```mysql
+    ALTER TABLE tb_name MODIFY COLUMN field_name data_type(length); 
+    ```
+    如果数据列的类型是可变长格式（BINARY 或 NCHAR），那么可以使用此指令修改其宽度（只能改大，不能改小）。（2.1.3.0 版本新增）  
+    如果表是通过超级表创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构。
+
 ## <a class="anchor" id="super-table"></a>超级表STable管理
 
 注意：在 2.0.15.0 及以后的版本中，开始支持 STABLE 保留字。也即，在本节后文的指令说明中，CREATE、DROP、ALTER 三个指令在老版本中保留字需写作 TABLE 而不是 STABLE。
@@ -323,6 +331,13 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传
     ALTER STABLE stb_name DROP COLUMN field_name; 
     ```
 
+- **超级表修改列宽**
+
+    ```mysql
+    ALTER STABLE stb_name MODIFY COLUMN field_name data_type(length); 
+    ```
+    如果数据列的类型是可变长格式（BINARY 或 NCHAR），那么可以使用此指令修改其宽度（只能改大，不能改小）。（2.1.3.0 版本新增）
+
 ## <a class="anchor" id="tags"></a>超级表 STable 中 TAG 管理
 
 - **添加标签**
@@ -346,6 +361,13 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传
     ```
     修改超级表的标签名，从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。
 
+- **修改标签列宽度**
+
+    ```mysql
+    ALTER STABLE stb_name MODIFY TAG tag_name data_type(length); 
+    ```
+    如果标签的类型是可变长格式（BINARY 或 NCHAR），那么可以使用此指令修改其宽度（只能改大，不能改小）。（2.1.3.0 版本新增）
+
 - **修改子表标签值**
 
     ```mysql

packaging/check_package.sh

+#!/bin/bash
+#
+# This file is used to install database on linux systems. The operating system
+# is required to use systemd to manage services at boot
+
+set -e
+#set -x
+
+verMode=edge
+pagMode=full
+
+iplist=""
+serverFqdn=""
+
+# -----------------------Variables definition---------------------
+script_dir="../release"
+# Dynamic directory
+data_dir="/var/lib/taos"
+log_dir="/var/log/taos"
+
+data_link_dir="/usr/local/taos/data"
+log_link_dir="/usr/local/taos/log"
+
+cfg_install_dir="/etc/taos"
+
+bin_link_dir="/usr/bin"
+lib_link_dir="/usr/lib"
+lib64_link_dir="/usr/lib64"
+inc_link_dir="/usr/include"
+
+#install main path
+install_main_dir="/usr/local/taos"
+
+# old bin dir
+sbin_dir="/usr/local/taos/bin"
+
+temp_version=""
+fin_result=""
+
+service_config_dir="/etc/systemd/system"
+nginx_port=6060
+nginx_dir="/usr/local/nginxd"
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+csudo=""
+if command -v sudo > /dev/null; then
+    csudo="sudo"
+fi
+
+# =============================  get input parameters =================================================
+
+# install.sh -v [server | client]  -e [yes | no] -i [systemd | service | ...]
+
+# set parameters by default value
+interactiveFqdn=yes   # [yes | no]
+verType=server        # [server | client]
+initType=systemd      # [systemd | service | ...]
+
+while getopts "hv:d:" arg
+do
+  case $arg in
+    d)
+      #echo "interactiveFqdn=$OPTARG"
+      script_dir=$( echo $OPTARG )
+      ;;
+    h)
+      echo "Usage: `basename $0` -d scripy_path"
+      exit 0
+      ;;
+    ?) #unknow option
+      echo "unkonw argument"
+      exit 1
+      ;;
+  esac
+done
+
+#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
+
+function kill_process() {
+  pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}')
+  if [ -n "$pid" ]; then
+    ${csudo} kill -9 $pid   || :
+  fi
+}
+
+function check_file() {
+    #check file whether exists
+    if [ ! -e $1/$2 ];then
+        echo -e "$1/$2 \033[31mnot exists\033[0m!quit"
+        fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n"
+        echo -e $fin_result
+        exit 8 
+    fi
+}
+
+function get_package_name() {
+  var=$1
+	if [[ $1 =~ 'aarch' ]];then
+		echo ${var::-21}
+	else
+		echo ${var::-17}
+	fi
+}
+function check_link() {
+    #check Link whether exists or broken
+    if [ -L $1 ] ; then
+        if [ ! -e $1 ] ; then
+            echo -e "$1 \033[31Broken link\033[0m"
+            fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n"
+            echo -e $fin_result
+            exit 8 
+        fi
+    else
+        echo -e "$1 \033[31mnot exists\033[0m!quit"
+        fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n"
+        echo -e $fin_result
+        exit 8 
+    fi
+}
+
+function check_main_path() {
+    #check install main dir and all sub dir
+    main_dir=("" "cfg" "bin" "connector" "driver" "examples" "include" "init.d")
+    for i in ${main_dir[@]};do
+        check_file ${install_main_dir} $i
+    done
+    if [ "$verMode" == "cluster" ]; then
+        nginx_main_dir=("admin" "conf" "html" "sbin" "logs")
+        for i in ${nginx_main_dir[@]};do
+            check_file ${nginx_dir}  $i
+        done
+    fi
+    echo -e "Check main path:\033[32mOK\033[0m!"
+}
+
+function check_bin_path() {
+    # check install bin dir and all sub dir
+    bin_dir=("taos" "taosd" "taosdemo" "taosdump" "remove.sh" "tarbitrator" "set_core.sh")
+    for i in ${bin_dir[@]};do
+        check_file ${sbin_dir} $i
+    done
+    lbin_dir=("taos" "taosd" "taosdemo" "taosdump" "rmtaos" "tarbitrator" "set_core")
+    for i in ${lbin_dir[@]};do
+        check_link ${bin_link_dir}/$i
+    done
+    if [ "$verMode" == "cluster" ]; then
+        check_file ${nginx_dir}/sbin nginx
+    fi
+    echo -e "Check bin  path:\033[32mOK\033[0m!"
+}
+
+
+function check_lib_path() {
+    # check all links
+    check_link ${lib_link_dir}/libtaos.so
+    check_link ${lib_link_dir}/libtaos.so.1
+
+    if [[ -d ${lib64_link_dir}  ]]; then
+        check_link ${lib64_link_dir}/libtaos.so
+        check_link ${lib64_link_dir}/libtaos.so.1
+    fi
+    echo -e "Check lib  path:\033[32mOK\033[0m!"
+}
+
+
+function check_header_path() {
+	# check all header 
+	header_dir=("taos.h" "taoserror.h")
+    for i in ${header_dir[@]};do
+        check_link ${inc_link_dir}/$i
+    done
+    echo -e "Check bin  path:\033[32mOK\033[0m!"
+}
+
+
+function check_config_dir() {
+	# check all config
+	check_file ${cfg_install_dir} taos.cfg
+	check_file ${install_main_dir}/cfg taos.cfg.org
+	echo -e "Check conf path:\033[32mOK\033[0m!"
+}
+
+function check_log_path() {
+	# check log path
+	check_file ${log_dir}
+	echo -e "Check log  path:\033[32mOK\033[0m!"
+}
+
+function check_data_path() {
+	# check data path
+	check_file ${data_dir} 
+	echo -e "Check data path:\033[32mOK\033[0m!"
+}
+
+function install_TDengine() {
+	cd ${script_dir}
+	tar zxf $1
+  temp_version=$(get_package_name $1)
+	cd $(get_package_name $1)
+  echo -e "\033[32muninstall TDengine && install TDengine...\033[0m"
+	rmtaos >/dev/null 2>&1 || echo 'taosd not installed' && echo -e '\n\n' |./install.sh >/dev/null 2>&1 
+  echo -e "\033[32mTDengine has been installed!\033[0m"
+  echo -e "\033[32mTDengine is starting...\033[0m"
+  kill_process taos && systemctl start taosd && sleep 10
+}
+
+function test_TDengine() {
+    check_main_path
+    check_bin_path
+    check_lib_path
+    check_header_path
+    check_config_dir
+    check_log_path
+    check_data_path
+    result=`taos -s 'create database test ;create table test.tt(ts timestamp ,i int);insert into test.tt values(now,11);select * from test.tt' 2>&1 ||:`
+    if [[ $result =~ "Unable to establish" ]];then
+        echo -e "\033[31mTDengine connect failed\033[0m" 
+        fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n"
+        echo -e $fin_result
+        exit 8 
+    fi  
+    echo -e "Check TDengine connect:\033[32mOK\033[0m!"
+    fin_result=$fin_result"\033[32m$temp_version\033[0m test OK!\n"
+} 
+# ## ==============================Main program starts from here============================
+TD_package_name=`ls ${script_dir}/*server*gz |awk -F '/' '{print $NF}' `
+temp=`pwd`
+for i in $TD_package_name;do
+	if [[ $i =~ 'enterprise' ]];then
+		verMode="cluster"
+	else
+		verMode=""
+	fi
+  cd $temp
+	install_TDengine $i
+	test_TDengine
+done
+echo "============================================================"
+echo -e $fin_result
\ No newline at end of file

src/client/src/tscSQLParser.c

@@ -953,8 +953,10 @@ int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pS
 static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, bool isStable) {
 
   const char* msg1 = "invalid column name";
+  const char* msg2 = "invalid column type";
   const char* msg3 = "not support state_window with group by ";
   const char* msg4 = "function not support for super table query";
+  const char* msg5 = "not support state_window on tag column";
 
   SStrToken *col = &(pSqlNode->windowstateVal.col) ;
   if (col->z == NULL || col->n <= 0) {
@@ -982,8 +984,10 @@ static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
   int32_t numOfCols = tscGetNumOfColumns(pTableMeta);
-  if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols) {
+  if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
     return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
+  } else if (index.columnIndex >= numOfCols) {
+    return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
   }
 
   SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr;
@@ -992,8 +996,10 @@ static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
   }
 
   SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex);
-  if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) {
-    return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
+  if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT 
+      || pSchema->type == TSDB_DATA_TYPE_DOUBLE || pSchema->type == TSDB_DATA_TYPE_NCHAR
+      || pSchema->type == TSDB_DATA_TYPE_BINARY) {
+    return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
   }
 
   tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema);

src/connector/python/taos/cinterface.py

@@ -15,7 +15,7 @@ def _convert_microsecond_to_datetime(micro):
 
 
 def _convert_nanosecond_to_datetime(nanosec):
-    return datetime.datetime.fromtimestamp(nanosec / 1000000000.0)
+    return nanosec
 
 
 def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):

src/os/inc/osFile.h

@@ -53,8 +53,24 @@ int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size
 void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath);
 void taosClose(FileFd fd);
 
+#ifdef TAOS_RANDOM_FILE_FAIL
+  void taosSetRandomFileFailFactor(int32_t factor);
+  void taosSetRandomFileFailOutput(const char *path);
+  #ifdef TAOS_RANDOM_FILE_FAIL_TEST
+    int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
+    int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
+    int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line);
+    #undef taosRead
+    #undef taosWrite
+    #undef taosLSeek
+    #define taosRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+    #define taosWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+    #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
+  #endif
+#endif
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif
\ No newline at end of file
+#endif

src/os/src/detail/osFail.c

@@ -113,7 +113,7 @@ int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char
     }
   }
 
-  return taosReadImp(fd, buf, count);
+  return taosRead(fd, buf, count);
 }
 
 int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) {
@@ -124,7 +124,7 @@ int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char
     }
   }
 
-  return taosWriteImp(fd, buf, count);
+  return taosWrite(fd, buf, count);
 }
 
 int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line) {
@@ -135,7 +135,7 @@ int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const ch
     }
   }
 
-  return taosLSeekImp(fd, offset, whence);
+  return taosLSeek(fd, offset, whence);
 }
 
 #endif //TAOS_RANDOM_FILE_FAIL

tests/pytest/crash_gen/valgrind_taos.supp

@@ -17517,4 +17517,209 @@
    fun:taosGetFqdn
    fun:taosCheckGlobalCfg
    fun:taos_init_imp
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__pyx_pymod_exec_mtrand
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   fun:PyCode_NewWithPosOnlyArgs
+   fun:PyCode_New
+   fun:__Pyx_InitCachedConstants
+   fun:__pyx_pymod_exec__generator
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__pyx_pymod_exec_bit_generator
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__pyx_pymod_exec__common
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__pyx_pymod_exec__bounded_integers
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__pyx_pymod_exec__mt19937
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__pyx_pymod_exec__philox
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__pyx_pymod_exec__pcg64
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__pyx_pymod_exec__sfc64
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   obj:/usr/bin/python3.8
+   fun:PyTuple_Pack
+   fun:__Pyx_InitCachedConstants
+   fun:__pyx_pymod_exec__generator
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   fun:PyCode_NewWithPosOnlyArgs
+   fun:PyCode_New
+   fun:__pyx_pymod_exec_mtrand
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   fun:PyCode_NewWithPosOnlyArgs
+   fun:PyCode_New
+   fun:__pyx_pymod_exec_bit_generator
+   fun:PyModule_ExecDef
+   obj:/usr/bin/python3.8
+   obj:/usr/bin/python3.8
+   fun:PyVectorcall_Call
+   fun:_PyEval_EvalFrameDefault
+   fun:_PyEval_EvalCodeWithName
+   fun:_PyFunction_Vectorcall
+   fun:_PyEval_EvalFrameDefault
 }
\ No newline at end of file

tests/pytest/dbmgmt/nanoSecondCheck.py

+# #################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+
+# #################################################################
+
+# -*- coding: utf-8 -*-
+
+# TODO: after TD-4518 and TD-4510 is resolved, add the exception test case for these situations
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+import time
+from datetime import datetime
+import os
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        tdSql.execute('reset query cache')
+        tdSql.execute('drop database if exists db')
+        tdSql.execute('create database db precision "ns";')
+        tdSql.query('show databases;')
+        tdSql.checkData(0,16,'ns')
+        tdSql.execute('use db')
+
+        tdLog.debug('testing nanosecond support in 1st timestamp')
+        tdSql.execute('create table tb (ts timestamp, speed int)')
+        tdSql.execute('insert into tb values(\'2021-06-10 0:00:00.100000001\', 1);')
+        tdSql.execute('insert into tb values(1623254400150000000, 2);')
+        tdSql.execute('import into tb values(1623254400300000000, 3);')
+        tdSql.execute('import into tb values(1623254400299999999, 4);')
+        tdSql.execute('insert into tb values(1623254400300000001, 5);')
+        tdSql.execute('insert into tb values(1623254400999999999, 7);')
+
+
+        tdSql.query('select * from tb;')
+        tdSql.checkData(0,0,'2021-06-10 0:00:00.100000001')
+        tdSql.checkData(1,0,'2021-06-10 0:00:00.150000000')
+        tdSql.checkData(2,0,'2021-06-10 0:00:00.299999999')
+        tdSql.checkData(3,1,3)
+        tdSql.checkData(4,1,5)
+        tdSql.checkData(5,1,7)
+        tdSql.checkRows(6)
+        tdSql.query('select count(*) from tb where ts > 1623254400100000000 and ts < 1623254400100000002;')
+        tdSql.checkData(0,0,1)
+        tdSql.query('select count(*) from tb where ts > \'2021-06-10 0:00:00.100000001\' and ts < \'2021-06-10 0:00:00.160000000\';')
+        tdSql.checkData(0,0,1)
+
+        tdSql.query('select count(*) from tb where ts > 1623254400100000000 and ts < 1623254400150000000;')
+        tdSql.checkData(0,0,1)
+        tdSql.query('select count(*) from tb where ts > \'2021-06-10 0:00:00.100000000\' and ts < \'2021-06-10 0:00:00.150000000\';')
+        tdSql.checkData(0,0,1)
+
+        tdSql.query('select count(*) from tb where ts > 1623254400400000000;')
+        tdSql.checkData(0,0,1)
+        tdSql.query('select count(*) from tb where ts < \'2021-06-10 00:00:00.400000000\';')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb where ts > now + 400000000b;')
+        tdSql.checkRows(0)
+
+        tdSql.query('select count(*) from tb where ts >= \'2021-06-10 0:00:00.100000001\';')
+        tdSql.checkData(0,0,6)
+
+        tdSql.query('select count(*) from tb where ts <= 1623254400300000000;')
+        tdSql.checkData(0,0,4)
+
+        tdSql.query('select count(*) from tb where ts = \'2021-06-10 0:00:00.000000000\';')
+        tdSql.checkRows(0)
+
+        tdSql.query('select count(*) from tb where ts = 1623254400150000000;')
+        tdSql.checkData(0,0,1)
+
+        tdSql.query('select count(*) from tb where ts = \'2021-06-10 0:00:00.100000001\';')
+        tdSql.checkData(0,0,1)
+
+        tdSql.query('select count(*) from tb where ts between 1623254400000000000 and 1623254400400000000;')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb where ts between \'2021-06-10 0:00:00.299999999\' and \'2021-06-10 0:00:00.300000001\';')
+        tdSql.checkData(0,0,3)
+
+        tdSql.query('select avg(speed) from tb interval(5000000000b);')
+        tdSql.checkRows(1)
+
+        tdSql.query('select avg(speed) from tb interval(100000000b)')
+        tdSql.checkRows(4)
+
+        tdSql.query('select avg(speed) from tb interval(100000000b) sliding (100000000b);')
+        tdSql.checkRows(4)
+
+        tdSql.query('select last(*) from tb')
+        tdSql.checkData(0,0, '2021-06-10 0:00:00.999999999')
+        tdSql.checkData(0,0, 1623254400999999999)
+
+        tdSql.query('select first(*) from tb')
+        tdSql.checkData(0,0, 1623254400100000001)
+        tdSql.checkData(0,0, '2021-06-10 0:00:00.100000001')
+
+        tdSql.execute('insert into tb values(now + 500000000b, 6);')
+        tdSql.query('select * from tb;')
+        tdSql.checkRows(7)
+
+        tdLog.debug('testing nanosecond support in other timestamps')
+        tdSql.execute('create table tb2 (ts timestamp, speed int, ts2 timestamp);')
+        tdSql.execute('insert into tb2 values(\'2021-06-10 0:00:00.100000001\', 1, \'2021-06-11 0:00:00.100000001\');')
+        tdSql.execute('insert into tb2 values(1623254400150000000, 2, 1623340800150000000);')
+        tdSql.execute('import into tb2 values(1623254400300000000, 3, 1623340800300000000);')
+        tdSql.execute('import into tb2 values(1623254400299999999, 4, 1623340800299999999);')
+        tdSql.execute('insert into tb2 values(1623254400300000001, 5, 1623340800300000001);')
+        tdSql.execute('insert into tb2 values(1623254400999999999, 7, 1623513600999999999);')
+
+        tdSql.query('select * from tb2;')
+        tdSql.checkData(0,0,'2021-06-10 0:00:00.100000001')
+        tdSql.checkData(1,0,'2021-06-10 0:00:00.150000000')
+        tdSql.checkData(2,1,4)
+        tdSql.checkData(3,1,3)
+        tdSql.checkData(4,2,'2021-06-11 00:00:00.300000001')
+        tdSql.checkData(5,2,'2021-06-13 00:00:00.999999999')
+        tdSql.checkRows(6)
+        tdSql.query('select count(*) from tb2 where ts2 > 1623340800000000000 and ts2 < 1623340800150000000;')
+        tdSql.checkData(0,0,1)
+        tdSql.query('select count(*) from tb2 where ts2 > \'2021-06-11 0:00:00.100000000\' and ts2 < \'2021-06-11 0:00:00.100000002\';')
+        tdSql.checkData(0,0,1)
+
+        tdSql.query('select count(*) from tb2 where ts2 > 1623340800500000000;')
+        tdSql.checkData(0,0,1)
+        tdSql.query('select count(*) from tb2 where ts2 < \'2021-06-11 0:00:00.400000000\';')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb2 where ts2 > now + 400000000b;')
+        tdSql.checkRows(0)
+
+        tdSql.query('select count(*) from tb2 where ts2 >= \'2021-06-11 0:00:00.100000001\';')
+        tdSql.checkData(0,0,6)
+
+        tdSql.query('select count(*) from tb2 where ts2 <= 1623340800400000000;')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb2 where ts2 = \'2021-06-11 0:00:00.000000000\';')
+        tdSql.checkRows(0)
+
+        tdSql.query('select count(*) from tb2 where ts2 = \'2021-06-11 0:00:00.300000001\';')
+        tdSql.checkData(0,0,1)
+
+        tdSql.query('select count(*) from tb2 where ts2 = 1623340800300000001;')
+        tdSql.checkData(0,0,1)
+
+        tdSql.query('select count(*) from tb2 where ts2 between 1623340800000000000 and 1623340800450000000;')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb2 where ts2 between \'2021-06-11 0:00:00.299999999\' and \'2021-06-11 0:00:00.300000001\';')
+        tdSql.checkData(0,0,3)
+
+        tdSql.query('select count(*) from tb2 where ts2 <> 1623513600999999999;')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb2 where ts2 <> \'2021-06-11 0:00:00.100000001\';')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb2 where ts2 <> \'2021-06-11 0:00:00.100000000\';')
+        tdSql.checkData(0,0,6)
+
+        tdSql.query('select count(*) from tb2 where ts2 != 1623513600999999999;')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb2 where ts2 != \'2021-06-11 0:00:00.100000001\';')
+        tdSql.checkData(0,0,5)
+
+        tdSql.query('select count(*) from tb2 where ts2 != \'2021-06-11 0:00:00.100000000\';')
+        tdSql.checkData(0,0,6)
+
+        tdSql.execute('insert into tb2 values(now + 500000000b, 6, now +2d);')
+        tdSql.query('select * from tb2;')
+        tdSql.checkRows(7)
+
+        tdLog.debug('testing ill nanosecond format handling')
+        tdSql.execute('create table tb3 (ts timestamp, speed int);')
+
+        tdSql.error('insert into tb3 values(16232544001500000, 2);')
+        tdSql.execute('insert into tb3 values(\'2021-06-10 0:00:00.123456\', 2);')
+        tdSql.query('select * from tb3 where ts = \'2021-06-10 0:00:00.123456000\';')
+        tdSql.checkRows(1)
+
+        tdSql.execute('insert into tb3 values(\'2021-06-10 0:00:00.123456789000\', 2);')
+        tdSql.query('select * from tb3 where ts = \'2021-06-10 0:00:00.123456789\';')
+        tdSql.checkRows(1)
+
+        os.system('sudo timedatectl set-ntp on')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/fulltest.sh

@@ -75,6 +75,7 @@ python3 ./test.py -f tag_lite/smallint.py
 python3 ./test.py -f tag_lite/tinyint.py
 
 #python3 ./test.py -f dbmgmt/database-name-boundary.py
+python3 test.py -f dbmgmt/nanoSecondCheck.py
 
 python3 ./test.py -f import_merge/importBlock1HO.py
 python3 ./test.py -f import_merge/importBlock1HPO.py

tests/pytest/functions/function_derivative.py

@@ -128,7 +128,14 @@ class TDTestCase:
 
     def run(self):
         tdSql.prepare()        
-        self.insertAndCheckData()               
+        self.insertAndCheckData()
+
+        tdSql.execute("create table st(ts timestamp, c1 int, c2 int) tags(id int)")
+        tdSql.execute("insert into dev1(ts, c1) using st tags(1) values(now, 1)")
+
+        tdSql.error("select derivative(c1, 10s, 0) from (select c1 from st)")
+        tdSql.query("select diff(c1) from (select derivative(c1, 1s, 0) c1 from dev1)")
+        tdSql.checkRows(0)
               
     def stop(self):
         tdSql.close()

tests/pytest/util/sql.py

@@ -18,6 +18,7 @@ import datetime
 import inspect
 import psutil
 import shutil
+import pandas as pd
 from util.log import *
 
 
@@ -134,25 +135,32 @@ class TDSql:
         return self.cursor.istype(col, dataType)
 
     def checkData(self, row, col, data):
-        self.checkRowCol(row, col) 
-        if self.queryResult[row][col] != data:            
-            if self.cursor.istype(col, "TIMESTAMP") and self.queryResult[row][col] == datetime.datetime.fromisoformat(data):
-                tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" %
+        self.checkRowCol(row, col)
+        if self.queryResult[row][col] != data:
+            if self.cursor.istype(col, "TIMESTAMP"):
+                # suppose user want to check nanosecond timestamp if a longer data passed
+                if (len(data) >= 28):
+                    if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data):
+                        tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" %
+                            (self.sql, row, col, self.queryResult[row][col], data))
+                else:
+                    if self.queryResult[row][col] == datetime.datetime.fromisoformat(data):
+                        tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" %
                             (self.sql, row, col, self.queryResult[row][col], data))
                 return
 
             if str(self.queryResult[row][col]) == str(data):
                 tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" %
-                            (self.sql, row, col, self.queryResult[row][col], data)) 
+                            (self.sql, row, col, self.queryResult[row][col], data))
                 return
-            elif isinstance(data, float) and abs(self.queryResult[row][col] - data) <= 0.000001:                
+            elif isinstance(data, float) and abs(self.queryResult[row][col] - data) <= 0.000001:
                 tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" %
-                            (self.sql, row, col, self.queryResult[row][col], data)) 
+                            (self.sql, row, col, self.queryResult[row][col], data))
                 return
             else:
                 caller = inspect.getframeinfo(inspect.stack()[1][0])
                 args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
-                tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)        
+                tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
 
         if data is None:
             tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" %
@@ -162,11 +170,11 @@ class TDSql:
                        (self.sql, row, col, self.queryResult[row][col], data))
         elif isinstance(data, datetime.date):
             tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" %
-                       (self.sql, row, col, self.queryResult[row][col], data))        
+                       (self.sql, row, col, self.queryResult[row][col], data))
         elif isinstance(data, float):
             tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" %
                        (self.sql, row, col, self.queryResult[row][col], data))
-        else:            
+        else:
             tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" %
                        (self.sql, row, col, self.queryResult[row][col], data))
 
@@ -200,7 +208,7 @@ class TDSql:
             tdLog.exit("%s(%d) failed: sql:%s, affectedRows:%d != expect:%d" % args)
 
         tdLog.info("sql:%s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows))
-    
+
     def taosdStatus(self, state):
         tdLog.sleep(5)
         pstate = 0
@@ -221,7 +229,7 @@ class TDSql:
                 continue
             pstate = 0
             break
-            
+
         args=(pstate,state)
         if pstate == state:
             tdLog.info("taosd state is %d == expect:%d" %args)
@@ -236,11 +244,11 @@ class TDSql:
                     tdLog.exit("dir: %s is empty, expect: not empty" %dir)
                 else:
                     tdLog.info("dir: %s is empty, expect: empty" %dir)
-            else:  
+            else:
                 if state :
                     tdLog.info("dir: %s is not empty, expect: not empty" %dir)
                 else:
-                    tdLog.exit("dir: %s is not empty, expect: empty" %dir)  
+                    tdLog.exit("dir: %s is not empty, expect: empty" %dir)
         else:
             tdLog.exit("dir: %s doesn't exist" %dir)
     def createDir(self, dir):
@@ -250,5 +258,5 @@ class TDSql:
         os.makedirs( dir, 755 )
         tdLog.info("dir: %s is created" %dir)
         pass
-        
+
 tdSql = TDSql()