[TS-530] merge master

.appveyor.yml

-version: 1.0.{build}
-image:
-  - Visual Studio 2015
-  - macos
-environment:
-  matrix:
-    - ARCH: amd64
-    - ARCH: x86
-matrix:
-  exclude:
-    - image: macos
-      ARCH: x86
-for:
-  -
-    matrix:
-      only:
-        - image: Visual Studio 2015
-    clone_folder: c:\dev\TDengine
-    clone_depth: 1
-
-    init:
-      - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %ARCH%
-
-    before_build:
-      - cd c:\dev\TDengine
-      - md build
-
-    build_script:
-      - cd build
-      - cmake -G "NMake Makefiles" .. -DBUILD_JDBC=false
-      - nmake install
-  -
-    matrix:
-      only:
-        - image: macos
-    clone_depth: 1
-
-    build_script:
-      - mkdir debug
-      - cd debug
-      - cmake .. > /dev/null
-      - make > /dev/null
-notifications:
-- provider: Email
-  to:
-  - sangshuduo@gmail.com
-  on_build_success: true
-  on_build_failure: true
-  on_build_status_changed: true

Jenkinsfile

@@ -38,7 +38,8 @@ def pre_test(){
     sudo rmtaos || echo "taosd has not installed"
     '''
     sh '''
-    killall -9 taosd ||echo "no taosd running"
+    kill -9 $(pidof taosd) ||echo "no taosd running"
+    kill -9 $(pidof taosadapter) ||echo "no taosadapter running"
     killall -9 gdb || echo "no gdb running"
     killall -9 python3.8 || echo "no python program running"
     cd ${WKC}

packaging/release.sh

@@ -349,8 +349,16 @@ if [[ "$dbName" == "kh" ]]; then
   sed -i "s/taos\.cfg/kinghistorian\.cfg/g"  ${top_dir}/src/util/src/tconfig.c
   sed -i "s/etc\/taos/etc\/kinghistorian/g"   ${top_dir}/src/util/src/tconfig.c
   # src/kit/taosdemo/taosdemo.c
+  sed -i "s/taosdemo --help/khdemo --help/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
+  sed -i "s/taosdemo --usage/khdemo --usage/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
+  sed -i "s/Usage: taosdemo/Usage: khdemo/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
+  sed -i "s/taosdemo is simulating/khdemo is simulating/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
+  sed -i "s/taosdemo version/khdemo version/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
   sed -i "s/\"taosdata\"/\"khroot\"/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
   sed -i "s/support@taosdata.com/support@wellintech.com/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
+  sed -i "s/taosc, rest, and stmt/khclient, rest, and stmt/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
+  sed -i "s/taosdemo uses/khdemo uses/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
+  sed -i "s/use 'taosc'/use 'khclient'/g" ${top_dir}/src/kit/taosdemo/taosdemo.c
   # src/util/src/tlog.c
   sed -i "s/log\/taos/log\/kinghistorian/g"   ${top_dir}/src/util/src/tlog.c
   # src/dnode/src/dnodeSystem.c
@@ -363,7 +371,7 @@ if [[ "$dbName" == "kh" ]]; then
   sed -i "s/taosinfo/khinfo/g"  ${top_dir}/src/util/src/tnote.c
   # src/dnode/CMakeLists.txt
   sed -i "s/taos\.cfg/kinghistorian\.cfg/g"  ${top_dir}/src/dnode/CMakeLists.txt
-  # src/dnode/CMakeLists.txt
+  # src/kit/taosdump/taosdump.c
   sed -i "s/Default is taosdata/Default is khroot/g" ${top_dir}/src/kit/taosdump/taosdump.c
   sed -i "s/\"taosdata\"/\"khroot\"/g" ${top_dir}/src/kit/taosdump/taosdump.c
   sed -i "s/TDengine/KingHistorian/g" ${top_dir}/src/kit/taosdump/taosdump.c
@@ -372,6 +380,12 @@ if [[ "$dbName" == "kh" ]]; then
   sed -i "s/etc\/taos/etc\/kinghistorian/g" ${top_dir}/src/os/src/linux/linuxEnv.c
   sed -i "s/lib\/taos/lib\/kinghistorian/g" ${top_dir}/src/os/src/linux/linuxEnv.c
   sed -i "s/log\/taos/log\/kinghistorian/g" ${top_dir}/src/os/src/linux/linuxEnv.c
+  # src/kit/shell/src/shellDarwin.c
+  sed -i "s/TDengine shell/KingHistorian shell/g" ${top_dir}/src/kit/shell/src/shellDarwin.c
+  sed -i "s/2020 by TAOS Data/2021 by Wellintech/g" ${top_dir}/src/kit/shell/src/shellDarwin.c
+  # src/kit/shell/src/shellLinux.c
+  sed -i "s/TDengine shell/KingHistorian shell/g" ${top_dir}/src/kit/shell/src/shellLinux.c
+  sed -i "s/2020 by TAOS Data/2021 by Wellintech/g" ${top_dir}/src/kit/shell/src/shellLinux.c
   # src/os/src/windows/wEnv.c
   sed -i "s/TDengine/KingHistorian/g" ${top_dir}/src/os/src/windows/wEnv.c
   # src/kit/shell/src/shellEngine.c

packaging/tools/install_client_jh.sh

@@ -188,9 +188,6 @@ function update() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config
@@ -215,9 +212,6 @@ function install() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config

packaging/tools/install_client_kh.sh

@@ -189,9 +189,6 @@ function update() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config
@@ -216,9 +213,6 @@ function install() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config

packaging/tools/install_client_power.sh

@@ -246,9 +246,6 @@ function update_PowerDB() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config
@@ -274,9 +271,6 @@ function install_PowerDB() {
     install_header
     install_lib
     install_jemalloc
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config

packaging/tools/install_client_pro.sh

@@ -188,9 +188,6 @@ function update_prodb() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config
@@ -215,9 +212,6 @@ function install_prodb() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config

packaging/tools/install_client_tq.sh

@@ -192,9 +192,6 @@ function update_tq() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config
@@ -219,9 +216,6 @@ function install_tq() {
     install_log
     install_header
     install_lib
-    if [ "$pagMode" != "lite" ]; then
-      install_connector
-    fi
     install_examples
     install_bin
     install_config

packaging/tools/makeclient.sh

@@ -46,7 +46,8 @@ if [ "$osType" != "Darwin" ]; then
     bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
   else
     bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \
-               ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb"
+               ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
+               #${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb"
   fi
   lib_files="${build_dir}/lib/libtaos.so.${version}"
 else

packaging/tools/makeclient_jh.sh

@@ -69,15 +69,15 @@ if [ "$osType" != "Darwin" ]; then
   if [ "$pagMode" == "lite" ]; then
     strip ${build_dir}/bin/taos
     cp ${build_dir}/bin/taos          ${install_dir}/bin/jh_taos
-    cp ${script_dir}/remove_jh.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_jh.sh  ${install_dir}/bin
   else 
     cp ${build_dir}/bin/taos          ${install_dir}/bin/jh_taos
-    cp ${script_dir}/remove_jh.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_jh.sh  ${install_dir}/bin
     cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/jhdemo
     cp ${build_dir}/bin/taosdump      ${install_dir}/bin/jh_taosdump
     cp ${script_dir}/set_core.sh      ${install_dir}/bin
     cp ${script_dir}/get_client.sh    ${install_dir}/bin
-    cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
+    #cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
   fi
 else
   cp ${bin_files} ${install_dir}/bin

packaging/tools/makeclient_kh.sh

@@ -70,15 +70,15 @@ if [ "$osType" != "Darwin" ]; then
   if [ "$pagMode" == "lite" ]; then
     strip ${build_dir}/bin/taos
     cp ${build_dir}/bin/taos          ${install_dir}/bin/khclient
-    cp ${script_dir}/remove_kh.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_kh.sh  ${install_dir}/bin
   else 
     cp ${build_dir}/bin/taos          ${install_dir}/bin/khclient
-    cp ${script_dir}/remove_kh.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_kh.sh  ${install_dir}/bin
     cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/khdemo
     cp ${build_dir}/bin/taosdump      ${install_dir}/bin/khdump
     cp ${script_dir}/set_core.sh      ${install_dir}/bin
     cp ${script_dir}/get_client.sh    ${install_dir}/bin
-    cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
+    #cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
   fi
 else
   cp ${bin_files} ${install_dir}/bin

packaging/tools/makeclient_power.sh

@@ -109,15 +109,15 @@ if [ "$osType" != "Darwin" ]; then
   if [ "$pagMode" == "lite" ]; then
     strip ${build_dir}/bin/taos
     cp ${build_dir}/bin/taos          ${install_dir}/bin/power
-    cp ${script_dir}/remove_power.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_power.sh  ${install_dir}/bin
   else
     cp ${build_dir}/bin/taos          ${install_dir}/bin/power
-    cp ${script_dir}/remove_power.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_power.sh  ${install_dir}/bin
     cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo
     cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump
     cp ${script_dir}/set_core.sh      ${install_dir}/bin
     cp ${script_dir}/get_client.sh    ${install_dir}/bin
-    cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
+    #cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
   fi
 else
   cp ${bin_files} ${install_dir}/bin

packaging/tools/makeclient_pro.sh

@@ -69,15 +69,15 @@ if [ "$osType" != "Darwin" ]; then
   if [ "$pagMode" == "lite" ]; then
     strip ${build_dir}/bin/taos
     cp ${build_dir}/bin/taos          ${install_dir}/bin/prodbc
-    cp ${script_dir}/remove_pro.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_pro.sh  ${install_dir}/bin
   else 
     cp ${build_dir}/bin/taos          ${install_dir}/bin/prodbc
-    cp ${script_dir}/remove_pro.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_pro.sh  ${install_dir}/bin
     cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/prodemo
     cp ${build_dir}/bin/taosdump      ${install_dir}/bin/prodump
     cp ${script_dir}/set_core.sh      ${install_dir}/bin
     cp ${script_dir}/get_client.sh    ${install_dir}/bin
-    cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
+    #cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
   fi
 else
   cp ${bin_files} ${install_dir}/bin

packaging/tools/makeclient_tq.sh

@@ -69,15 +69,15 @@ if [ "$osType" != "Darwin" ]; then
   if [ "$pagMode" == "lite" ]; then
     strip ${build_dir}/bin/taos
     cp ${build_dir}/bin/taos          ${install_dir}/bin/tq
-    cp ${script_dir}/remove_tq.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_tq.sh  ${install_dir}/bin
   else 
     cp ${build_dir}/bin/taos          ${install_dir}/bin/tq
-    cp ${script_dir}/remove_tq.sh  ${install_dir}/bin
+    cp ${script_dir}/remove_client_tq.sh  ${install_dir}/bin
     cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/tqdemo
     cp ${build_dir}/bin/taosdump      ${install_dir}/bin/tqdump
     cp ${script_dir}/set_core.sh      ${install_dir}/bin
     cp ${script_dir}/get_client.sh    ${install_dir}/bin
-    cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
+    #cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
   fi
 else
   cp ${bin_files} ${install_dir}/bin

src/client/inc/tsclient.h

@@ -115,8 +115,9 @@ typedef struct SParsedDataColInfo {
   int16_t        numOfCols;
   int16_t        numOfBound;
   uint16_t       flen;        // TODO: get from STSchema
-  uint16_t       allNullLen;  // TODO: get from STSchema
+  uint16_t       allNullLen;  // TODO: get from STSchema(base on SDataRow)
   uint16_t       extendedVarLen;
+  uint16_t       boundNullLen;    // bound column len with all NULL value(without VarDataOffsetT/SColIdx part)
   int32_t *      boundedColumns;  // bound column idx according to schema
   SBoundColumn * cols;
   SBoundIdxInfo *colIdxInfo;
@@ -132,7 +133,7 @@ typedef struct {
 typedef struct {
   uint8_t      memRowType;   // default is 0, that is SDataRow 
   uint8_t      compareStat;  // 0 no need, 1 need compare
-  TDRowTLenT   kvRowInitLen;
+  int32_t      rowSize;
   SMemRowInfo *rowInfo;
 } SMemRowBuilder;
 
@@ -143,8 +144,7 @@ typedef enum {
 
 int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec);
 
-int  initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols,
-                       int32_t allNullLen);
+int  initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo);
 void destroyMemRowBuilder(SMemRowBuilder *pBuilder);
 
 /**
@@ -535,16 +535,6 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
   return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen;
 }
 
-static FORCE_INLINE void checkAndConvertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) {
-  if (isDataRow(row)) {
-    if (kvLen < (dataLen * KVRatioConvert)) {
-      memRowSetConvert(row);
-    }
-  } else if (kvLen > dataLen) {
-    memRowSetConvert(row);
-  }
-}
-
 static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) {
   memRowSetType(row, memRowType);
   if (isDataRowT(memRowType)) {
@@ -644,8 +634,7 @@ static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
 static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
 
 static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, SMemRow row, char *msg, char **str,
-                                               bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId,
-                                               int32_t *dataLen, int32_t *kvLen, uint8_t compareStat) {
+                                               bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId) {
   int64_t iv;
   int32_t ret;
   char *  endptr = NULL;
@@ -657,26 +646,22 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
   switch (pSchema->type) {
     case TSDB_DATA_TYPE_BOOL: {  // bool
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) {
           if (strncmp(pToken->z, "true", pToken->n) == 0) {
-            tscAppendMemRowColValEx(row, &TRUE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+            tdAppendMemRowColVal(row, &TRUE_VALUE, true, colId, pSchema->type, toffset);
           } else if (strncmp(pToken->z, "false", pToken->n) == 0) {
-            tscAppendMemRowColValEx(row, &FALSE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                    compareStat);
+            tdAppendMemRowColVal(row, &FALSE_VALUE, true, colId, pSchema->type, toffset);
           } else {
             return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z);
           }
         } else if (pToken->type == TK_INTEGER) {
           iv = strtoll(pToken->z, NULL, 10);
-          tscAppendMemRowColValEx(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset,
-                                  dataLen, kvLen, compareStat);
+          tdAppendMemRowColVal(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset);
         } else if (pToken->type == TK_FLOAT) {
           double dv = strtod(pToken->z, NULL);
-          tscAppendMemRowColValEx(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset,
-                                  dataLen, kvLen, compareStat);
+          tdAppendMemRowColVal(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset);
         } else {
           return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z);
         }
@@ -686,8 +671,7 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
 
     case TSDB_DATA_TYPE_TINYINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -697,15 +681,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint8_t tmpVal = (uint8_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_UTINYINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -715,15 +698,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint8_t tmpVal = (uint8_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_SMALLINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -733,15 +715,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         int16_t tmpVal = (int16_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_USMALLINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -751,15 +732,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint16_t tmpVal = (uint16_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_INT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -769,15 +749,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         int32_t tmpVal = (int32_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_UINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -787,15 +766,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint32_t tmpVal = (uint32_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_BIGINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -804,14 +782,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
           return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z);
         }
 
-        tscAppendMemRowColValEx(row, &iv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &iv, true, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_UBIGINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -821,14 +798,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint64_t tmpVal = (uint64_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_FLOAT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         double dv;
         if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
@@ -841,14 +817,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         float tmpVal = (float)dv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_DOUBLE:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         double dv;
         if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
@@ -859,15 +834,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
           return tscInvalidOperationMsg(msg, "illegal double data", pToken->z);
         }
 
-        tscAppendMemRowColValEx(row, &dv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &dv, true, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_BINARY:
       // binary data cannot be null-terminated char string, otherwise the last char of the string is lost
       if (pToken->type == TK_NULL) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {  // too long values will return invalid sql, not be truncated automatically
         if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {  // todo refactor
           return tscInvalidOperationMsg(msg, "string data overflow", pToken->z);
@@ -875,14 +849,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n);
         char *rowEnd = memRowEnd(row);
         STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n);
-        tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_NCHAR:
       if (pToken->type == TK_NULL) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
         int32_t output = 0;
@@ -894,7 +867,7 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
           return tscInvalidOperationMsg(msg, buf, pToken->z);
         }
         varDataSetLen(rowEnd, output);
-        tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset);
       }
       break;
 
@@ -903,17 +876,16 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         if (primaryKey) {
           // When building SKVRow primaryKey, we should not skip even with NULL value.
           int64_t tmpVal = 0;
-          tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+          tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
         } else {
-          tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                  compareStat);
+          tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
         }
       } else {
         int64_t tmpVal;
         if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) {
           return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z);
         }
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;

src/client/src/tscParseInsert.c

@@ -41,9 +41,8 @@ enum {
 static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows);
 static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema,
                                  char *str, char **end);
-int            initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols,
-                                 int32_t allNullLen) {
-  ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols));
+int            initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo) {
+  ASSERT(nRows >= 0 && pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols));
   if (nRows > 0) {
     // already init(bind multiple rows by single column)
     if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) {
@@ -51,41 +50,12 @@ int            initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint3
     }
   }
 
-  // default compareStat is  ROW_COMPARE_NO_NEED
-  if (nBoundCols == 0) {  // file input
-    pBuilder->memRowType = SMEM_ROW_DATA;
-    return TSDB_CODE_SUCCESS;
+  uint32_t dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + pColInfo->allNullLen;
+  uint32_t kvLen = TD_MEM_ROW_KV_HEAD_SIZE + pColInfo->numOfBound * sizeof(SColIdx) + pColInfo->boundNullLen;
+  if (isUtilizeKVRow(kvLen, dataLen)) {
+    pBuilder->memRowType = SMEM_ROW_KV;
   } else {
-    float boundRatio = ((float)nBoundCols / (float)nCols);
-
-    if (boundRatio < KVRatioKV) {
-      pBuilder->memRowType = SMEM_ROW_KV;
-      return TSDB_CODE_SUCCESS;
-    } else if (boundRatio > KVRatioData) {
-      pBuilder->memRowType = SMEM_ROW_DATA;
-      return TSDB_CODE_SUCCESS;
-    }
-    pBuilder->compareStat = ROW_COMPARE_NEED;
-
-    if (boundRatio < KVRatioPredict) {
-      pBuilder->memRowType = SMEM_ROW_KV;
-    } else {
-      pBuilder->memRowType = SMEM_ROW_DATA;
-    }
-  }
-
-  pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx);
-
-  if (nRows > 0) {
-    pBuilder->rowInfo = tcalloc(nRows, sizeof(SMemRowInfo));
-    if (pBuilder->rowInfo == NULL) {
-      return TSDB_CODE_TSC_OUT_OF_MEMORY;
-    }
-
-    for (int i = 0; i < nRows; ++i) {
-      (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
-      (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen;
-    }
+    pBuilder->memRowType = SMEM_ROW_DATA;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -457,8 +427,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
   STableMeta *        pTableMeta = pDataBlocks->pTableMeta;
   SSchema *           schema = tscGetTableSchema(pTableMeta);
   SMemRowBuilder *    pBuilder = &pDataBlocks->rowBuilder;
-  int32_t             dataLen = spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE;
-  int32_t             kvLen = pBuilder->kvRowInitLen;
   bool                isParseBindParam = false;
 
   initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound);
@@ -535,8 +503,8 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
     int16_t colId = -1;
     tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId);
 
-    int32_t ret = tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset,
-                                     colId, &dataLen, &kvLen, pBuilder->compareStat);
+    int32_t ret =
+        tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
     }
@@ -551,13 +519,8 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
   }
 
   if (!isParseBindParam) {
-    // 2. check and set convert flag
-    if (pBuilder->compareStat == ROW_COMPARE_NEED) {
-      checkAndConvertMemRow(row, dataLen, kvLen);
-    }
-
-    // 3. set the null value for the columns that do not assign values
-    if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) {
+    // set the null value for the columns that do not assign values
+    if ((spd->numOfBound < spd->numOfCols) && isDataRow(row)) {
       SDataRow dataRow = memRowDataBody(row);
       for (int32_t i = 0; i < spd->numOfCols; ++i) {
         if (spd->cols[i].valStat == VAL_STAT_NONE) {
@@ -567,7 +530,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
     }
   }
 
-  *len = getExtendedRowSize(pDataBlocks);
+  *len = pBuilder->rowSize;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -620,11 +583,11 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn
 
   int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
 
-  if (TSDB_CODE_SUCCESS !=
-      (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound,
-                                pDataBlock->boundColumnInfo.allNullLen))) {
+  if (TSDB_CODE_SUCCESS != (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, &pDataBlock->boundColumnInfo))) {
     return code;
   }
+  pDataBlock->rowBuilder.rowSize = extendedRowSize;
+
   while (1) {
     tsc_index = 0;
     sToken = tStrGetToken(*str, &tsc_index, false);
@@ -703,6 +666,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32
     pColInfo->boundedColumns[i] = i;
   }
   pColInfo->allNullLen += pColInfo->flen;
+  pColInfo->boundNullLen = pColInfo->allNullLen;  // default set allNullLen
   pColInfo->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT));
 }
 
@@ -1206,6 +1170,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
   int32_t nCols = pColInfo->numOfCols;
 
   pColInfo->numOfBound = 0;
+  pColInfo->boundNullLen = 0;
   memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols);
   for (int32_t i = 0; i < nCols; ++i) {
     pColInfo->cols[i].valStat = VAL_STAT_NONE;
@@ -1255,6 +1220,17 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
         pColInfo->cols[t].valStat = VAL_STAT_HAS;
         pColInfo->boundedColumns[pColInfo->numOfBound] = t;
         ++pColInfo->numOfBound;
+        switch (pSchema[t].type) {
+          case TSDB_DATA_TYPE_BINARY:
+            pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES);
+            break;
+          case TSDB_DATA_TYPE_NCHAR:
+            pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
+            break;
+          default:
+            pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type];
+            break;
+        }        
         findColumnIndex = true;
         if (isOrdered && (lastColIdx > t)) {
           isOrdered = false;
@@ -1278,6 +1254,17 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
           pColInfo->cols[t].valStat = VAL_STAT_HAS;
           pColInfo->boundedColumns[pColInfo->numOfBound] = t;
           ++pColInfo->numOfBound;
+          switch (pSchema[t].type) {
+            case TSDB_DATA_TYPE_BINARY:
+              pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES);
+              break;
+            case TSDB_DATA_TYPE_NCHAR:
+              pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
+              break;
+            default:
+              pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type];
+              break;
+          }          
           findColumnIndex = true;
           if (isOrdered && (lastColIdx > t)) {
             isOrdered = false;
@@ -1722,13 +1709,18 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow
     goto _error;
   }
 
-  tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(pTableDataBlock), &maxRows);
+  int32_t extendedRowSize = getExtendedRowSize(pTableDataBlock);
+  tscAllocateMemIfNeed(pTableDataBlock, extendedRowSize, &maxRows);
   tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW);
   if (tokenBuf == NULL) {
     code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     goto _error;
   }
 
+ // insert from .csv means full and ordered columns, thus use SDataRow all the time
+  ASSERT(SMEM_ROW_DATA == pTableDataBlock->rowBuilder.memRowType);
+  pTableDataBlock->rowBuilder.rowSize = extendedRowSize;
+  
   while ((readLen = tgetline(&line, &n, fp)) != -1) {
     if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
       line[--readLen] = 0;

src/client/src/tscPrepare.c

@@ -1527,6 +1527,41 @@ int stmtGenInsertStatement(SSqlObj* pSql, STscStmt* pStmt, const char* name, TAO
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t stmtValidateValuesFields(SSqlCmd *pCmd, char * sql) {
+  int32_t loopCont = 1, index0 = 0, values = 0;
+  SStrToken sToken;
+  
+  while (loopCont) {
+    sToken = tStrGetToken(sql, &index0, false);
+    if (sToken.n <= 0) {
+      return TSDB_CODE_SUCCESS;
+    }
+  
+    switch (sToken.type) {
+      case TK_RP:
+        if (values) {
+         return TSDB_CODE_SUCCESS;
+        }
+        break;
+      case TK_VALUES:
+        values = 1;
+        break;
+      case TK_QUESTION:  
+      case TK_LP:
+        break;
+      default:
+        if (values) {
+          tscError("only ? allowed in values");
+          return tscInvalidOperationMsg(pCmd->payload, "only ? allowed in values", NULL);
+        }
+        break;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 ////////////////////////////////////////////////////////////////////////////////
 // interface functions
 
@@ -1631,6 +1666,11 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
       STMT_RET(ret);
     }
 
+    ret = stmtValidateValuesFields(&pSql->cmd, pSql->sqlstr);
+    if (ret != TSDB_CODE_SUCCESS) {
+      STMT_RET(ret);
+    }
+
     if (pStmt->multiTbInsert) {
       STMT_RET(TSDB_CODE_SUCCESS);
     }

src/client/src/tscSQLParser.c

@@ -99,7 +99,7 @@ static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
 
 static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery);
 
-static int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql);
+static int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql, bool joinQuery);
 static int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode);
 static int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema);
 
@@ -4483,7 +4483,7 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
 }
 
 static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr,
-                                     int32_t* type, int32_t parentOptr, int32_t* tbIdx) {
+                                     int32_t* type, int32_t parentOptr, int32_t* tbIdx, bool joinQuery) {
   const char* msg1 = "table query cannot use tags filter";
   const char* msg2 = "illegal column name";
   const char* msg3 = "only one query time range allowed";
@@ -4491,6 +4491,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   const char* msg5 = "not support ordinary column join";
   const char* msg6 = "only one query condition on tbname allowed";
   const char* msg7 = "only in/like allowed in filter table name";
+  const char* msg8 = "illegal condition expression";
 
   tSqlExpr* pLeft  = (*pExpr)->pLeft;
   tSqlExpr* pRight = (*pExpr)->pRight;
@@ -4630,8 +4631,12 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   } else {  // query on other columns
     *type = TSQL_EXPR_COLUMN;
 
-    if (pRight->tokenId == TK_ID) {  // other column cannot be served as the join column
-      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
+    if (pRight->tokenId == TK_ID) {
+      if (joinQuery) {
+        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); // other column cannot be served as the join column
+      } else {
+        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8);
+      }
     }
 
     ret = setExprToCond(&pCondExpr->pColumnCond, *pExpr, NULL, parentOptr, pCmd->payload);
@@ -4642,7 +4647,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
 }
 
 int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr,
-                        int32_t* type, int32_t parentOptr, int32_t *tbIdx) {
+                        int32_t* type, int32_t parentOptr, int32_t *tbIdx, bool joinQuery) {
   if (pExpr == NULL) {
     return TSDB_CODE_SUCCESS;
   }
@@ -4666,12 +4671,12 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr
   int32_t rightTbIdx = 0;
 
   if (!tSqlExprIsParentOfLeaf(*pExpr)) {
-    int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->tokenId, &leftTbIdx);
+    int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->tokenId, &leftTbIdx, joinQuery);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
     }
 
-    ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->tokenId, &rightTbIdx);
+    ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->tokenId, &rightTbIdx, joinQuery);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
     }
@@ -4693,7 +4698,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr
     *type = rightType;
 
     *tbIdx = (leftTbIdx == rightTbIdx) ? leftTbIdx : -1;
-    
+
     return TSDB_CODE_SUCCESS;
   }
 
@@ -4707,7 +4712,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr
     return TSDB_CODE_TSC_INVALID_OPERATION;
   }
 
-  return handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, parentOptr, tbIdx);
+  return handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, parentOptr, tbIdx, joinQuery);
 }
 
 static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, tSqlExpr** pOut, int32_t tableIndex) {
@@ -5233,11 +5238,11 @@ int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) {
 }
 
 
-int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) {
+int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql, bool joinQuery) {
   if (pExpr == NULL) {
     return TSDB_CODE_SUCCESS;
   }
-  
+
   const char* msg1 = "invalid expression";
   const char* msg2 = "invalid filter expression";
 
@@ -5253,7 +5258,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq
 
   int32_t type = 0;
   int32_t tbIdx = 0;
-  if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId, &tbIdx)) != TSDB_CODE_SUCCESS) {
+  if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId, &tbIdx, joinQuery)) != TSDB_CODE_SUCCESS) {
     goto PARSE_WHERE_EXIT;
   }
 
@@ -5271,7 +5276,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq
   if ((ret = getTimeRangeFromExpr(&pSql->cmd, pQueryInfo, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) {
     return ret;
   }
-  
+
 
   // 3. get the tag query condition
   if ((ret = getTagQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr, pExpr)) != TSDB_CODE_SUCCESS) {
@@ -7803,11 +7808,12 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
     }
   }  
   
+
   SRelationInfo* pFromInfo = pInfo->pCreateTableInfo->pSelect->from;
   if (pFromInfo == NULL || taosArrayGetSize(pFromInfo->list) == 0) {
     return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
   }
-  
+
   SRelElementPair* p1 = taosArrayGet(pFromInfo->list, 0);
   SStrToken srcToken = {.z = p1->tableName.z, .n = p1->tableName.n, .type = TK_STRING};
   if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) {
@@ -7828,8 +7834,10 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
     return TSDB_CODE_TSC_INVALID_OPERATION;
   }
 
+  int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1);
+
   if (pSqlNode->pWhere != NULL) {  // query condition in stream computing
-    if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) {
+    if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql, joinQuery) != TSDB_CODE_SUCCESS) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
   }
@@ -8760,6 +8768,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
         (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap));
     TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY);
 
+    int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1);
+
     // parse the group by clause in the first place
     if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
@@ -8804,7 +8814,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
 
     // validate the query filter condition info
     if (pSqlNode->pWhere != NULL) {
-      if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) {
+      if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql, joinQuery) != TSDB_CODE_SUCCESS) {
         return TSDB_CODE_TSC_INVALID_OPERATION;
       }
     } else {
@@ -8849,7 +8859,6 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
     }
 
     // parse the having clause in the first place
-    int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1);
     if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) !=
         TSDB_CODE_SUCCESS) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
@@ -8897,6 +8906,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
     int32_t type = isSTable? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_TABLE_QUERY;
     TSDB_QUERY_SET_TYPE(pQueryInfo->type, type);
 
+    int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1);
+
     // parse the group by clause in the first place
     if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
@@ -8904,7 +8915,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
     pQueryInfo->onlyHasTagCond = true;
     // set where info
     if (pSqlNode->pWhere != NULL) {
-      if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) {
+      if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql, joinQuery) != TSDB_CODE_SUCCESS) {
         return TSDB_CODE_TSC_INVALID_OPERATION;
       }
 
@@ -8915,7 +8926,6 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
       }
     }
 
-    int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1);
     int32_t timeWindowQuery =
         (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap));
 

src/client/src/tscUtil.c

@@ -1943,18 +1943,11 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI
     }
   } else {
     for (int32_t i = 0; i < numOfRows; ++i) {
-      char* payload = (blkKeyTuple + i)->payloadAddr;
-      if (isNeedConvertRow(payload)) {
-        convertSMemRow(pDataBlock, payload, pTableDataBlock);
-        TDRowTLenT rowTLen = memRowTLen(pDataBlock);
-        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
-        pBlock->dataLen += rowTLen;
-      } else {
-        TDRowTLenT rowTLen = memRowTLen(payload);
-        memcpy(pDataBlock, payload, rowTLen);
-        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
-        pBlock->dataLen += rowTLen;
-      }
+      char*      payload = (blkKeyTuple + i)->payloadAddr;
+      TDRowTLenT rowTLen = memRowTLen(payload);
+      memcpy(pDataBlock, payload, rowTLen);
+      pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
+      pBlock->dataLen += rowTLen;
     }
   }
 

src/common/inc/tdataformat.h

@@ -609,22 +609,17 @@ typedef void *SMemRow;
 
 #define SMEM_ROW_DATA 0x0U      // SDataRow
 #define SMEM_ROW_KV 0x01U       // SKVRow
-#define SMEM_ROW_CONVERT 0x80U  // SMemRow convert flag
 
-#define KVRatioKV (0.2f)  // all bool
-#define KVRatioPredict (0.4f)
-#define KVRatioData (0.75f)  // all bigint
 #define KVRatioConvert (0.9f)
 
 #define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
 
 #define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t))  // set the total byte in case of dirty memory
-#define memRowSetConvert(r) ((*(uint8_t *)(r)) = (((*(uint8_t *)(r)) & 0x7F) | SMEM_ROW_CONVERT))  // highest bit
 #define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
 #define isDataRow(r) (SMEM_ROW_DATA == memRowType(r))
 #define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
 #define isKvRow(r) (SMEM_ROW_KV == memRowType(r))
-#define isNeedConvertRow(r) (((*(uint8_t *)(r)) & 0x80) == SMEM_ROW_CONVERT)
+#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert))
 
 #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)  // section after flag
 #define memRowKvBody(r) \

src/common/inc/tvariant.h

@@ -41,6 +41,8 @@ bool tVariantIsValid(tVariant *pVar);
 
 void tVariantCreate(tVariant *pVar, SStrToken *token);
 
+void tVariantCreateExt(tVariant *pVar, SStrToken *token, int32_t optrType);
+
 void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type);
 
 void tVariantDestroy(tVariant *pV);

src/common/src/tvariant.c

@@ -16,6 +16,7 @@
 
 #include "hash.h"
 #include "taos.h"
+#include "taoserror.h"
 #include "taosdef.h"
 #include "ttoken.h"
 #include "ttokendef.h"
@@ -24,6 +25,10 @@
 #include "tvariant.h"
 
 void tVariantCreate(tVariant *pVar, SStrToken *token) {
+  tVariantCreateExt(pVar, token, TK_ID);
+}
+
+void tVariantCreateExt(tVariant *pVar, SStrToken *token, int32_t optrType) {
   int32_t ret = 0;
   int32_t type = token->type;
 
@@ -47,7 +52,7 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) {
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_INT:{
       ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true);
-      if (ret != 0) {
+      if (ret != TSDB_CODE_SUCCESS) {
         SStrToken t = {0};
         tGetToken(token->z, &t.type);
         if (t.type == TK_MINUS) {  // it is a signed number which is greater than INT64_MAX or less than INT64_MIN
@@ -57,7 +62,7 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) {
 
         // data overflow, try unsigned parse the input number
         ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, false);
-        if (ret != 0) {
+        if (ret != TSDB_CODE_SUCCESS) {
           pVar->nType = -1;   // -1 means error type
           return;
         }
@@ -78,15 +83,29 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) {
       break;
     }
     case TSDB_DATA_TYPE_TIMESTAMP: {
-      pVar->i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO);                           
-      break;                             
-    }                            
-    
+      if (optrType == TK_NOW) {
+        pVar->i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
+      } else if (optrType == TK_PLUS || optrType == TK_MINUS) {
+        char unit = 0;
+        ret = parseAbsoluteDuration(token->z, token->n, &pVar->i64, &unit, TSDB_TIME_PRECISION_NANO);
+        if (ret != TSDB_CODE_SUCCESS) {
+          pVar->nType = -1;   // -1 means error type
+          return;
+        }
+        if (optrType == TK_PLUS) {
+          pVar->i64 += taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
+        } else {
+          pVar->i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO) - pVar->i64;
+        }
+      }
+      break;
+    }
+
     default: {  // nType == 0 means the null value
       type = TSDB_DATA_TYPE_NULL;
     }
   }
-  
+
   pVar->nType = type;
 }
 
@@ -157,7 +176,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
       pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE);
       memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE);
       pVar->nLen = (int32_t)len;
-      
+
       break;
     }
     case TSDB_DATA_TYPE_BINARY: {  // todo refactor, extract a method
@@ -166,12 +185,12 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
       pVar->nLen = (int32_t)len;
       break;
     }
-    
+
     default:
       pVar->i64 = GET_INT32_VAL(pz);
       pVar->nLen = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
   }
-  
+
   pVar->nType = type;
 }
 

src/inc/ttokendef.h

@@ -215,7 +215,6 @@
 #define TK_VALUES                         197
 #define TK_FILE                           198
 
-
 #define TK_SPACE                          300
 #define TK_COMMENT                        301
 #define TK_ILLEGAL                        302

src/kit/shell/src/shellDarwin.c

@@ -34,10 +34,10 @@ void insertChar(Command *cmd, char *c, int size);
 
 void printHelp() {
   char indent[10] = "        ";
-  printf("taos shell is used to test the TDengine database\n");
+  printf("taos shell is used to test the database\n");
 
   printf("%s%s\n", indent, "-h");
-  printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost.");
+  printf("%s%s%s\n", indent, indent, "Server IP address to connect. The default host is localhost.");
   printf("%s%s\n", indent, "-p");
   printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server.");
   printf("%s%s\n", indent, "-P");

src/kit/shell/src/shellLinux.c

@@ -37,8 +37,8 @@ const char *argp_program_bug_address = "<support@taosdata.com>";
 static char doc[] = "";
 static char args_doc[] = "";
 static struct argp_option options[] = {
-  {"host",       'h', "HOST",       0,                   "TDengine server FQDN to connect. The default host is localhost."},
-  {"password",   'p', 0,   0,                   "The password to use when connecting to the server."},
+  {"host",       'h', "HOST",       0,                   "Server FQDN to connect. The default host is localhost."},
+  {"password",   'p', 0,            0,                   "The password to use when connecting to the server."},
   {"port",       'P', "PORT",       0,                   "The TCP/IP port number to use for the connection."},
   {"user",       'u', "USER",       0,                   "The user name to use when connecting to the server."},
   {"auth",       'A', "Auth",       0,                   "The auth string to use when connecting to the server."},

src/kit/shell/src/shellWindows.c

@@ -21,8 +21,26 @@
 
 extern char configDir[];
 
+#ifdef _TD_POWER_
+char      WINCLIENT_VERSION[] = "Welcome to the PowerDB shell from %s, Client Version:%s\n"
+                             "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n";
+#elif (_TD_TQ_ == true)
+char      WINCLIENT_VERSION[] = "Welcome to the TQ shell from %s, Client Version:%s\n"
+                             "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
+#elif (_TD_PRO_ == true)
+char      WINCLIENT_VERSION[] = "Welcome to the ProDB shell from %s, Client Version:%s\n"
+                             "Copyright (c) 2020 by Hanatech, Inc. All rights reserved.\n\n";
+#elif (_TD_KH_ == true)
+char      WINCLIENT_VERSION[] = "Welcome to the KingHistorian shell from %s, Client Version:%s\n"
+                             "Copyright (c) 2021 by Wellintech, Inc. All rights reserved.\n\n";
+#elif (_TD_JH_ == true)
+char      WINCLIENT_VERSION[] = "Welcome to the jh_iot shell from %s, Client Version:%s\n"
+                             "Copyright (c) 2021 by jinheng, Inc. All rights reserved.\n\n";
+#else
 char      WINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
                              "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
+#endif
+
 
 void printVersion() {
   printf("version: %s\n", version);
@@ -30,10 +48,10 @@ void printVersion() {
 
 void printHelp() {
   char indent[10] = "        ";
-  printf("taos shell is used to test the TDengine database\n");
+  printf("taos shell is used to test the database\n");
 
   printf("%s%s\n", indent, "-h");
-  printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost.");
+  printf("%s%s%s\n", indent, indent, "Server FQDN to connect. The default host is localhost.");
   printf("%s%s\n", indent, "-p");
   printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server.");
   printf("%s%s\n", indent, "-P");

src/kit/taosdemo/taosdemo.c

@@ -773,7 +773,7 @@ static void printHelp() {
     printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t",
             "Configuration directory.");
     printf("%s%s%s%s\n", indent, "-h, --host=HOST", "\t\t",
-            "TDengine server FQDN to connect. The default host is localhost.");
+            "Server FQDN to connect. The default host is localhost.");
     printf("%s%s%s%s\n", indent, "-P, --port=PORT", "\t\t",
             "The TCP/IP port number to use for the connection.");
     printf("%s%s%s%s\n", indent, "-I, --interface=INTERFACE", "\t",
@@ -10963,8 +10963,6 @@ static int insertTestProcess() {
     assert(cmdBuffer);
 
     if(createDatabasesAndStables(cmdBuffer) != 0) {
-        if (g_fpOfInsertResult)
-            fclose(g_fpOfInsertResult);
         free(cmdBuffer);
         return -1;
     }

src/query/inc/sql.y

@@ -459,7 +459,17 @@ tagitem(A) ::= FLOAT(X).        { toTSDBType(X.type); tVariantCreate(&A, &X); }
 tagitem(A) ::= STRING(X).       { toTSDBType(X.type); tVariantCreate(&A, &X); }
 tagitem(A) ::= BOOL(X).         { toTSDBType(X.type); tVariantCreate(&A, &X); }
 tagitem(A) ::= NULL(X).         { X.type = 0; tVariantCreate(&A, &X); }
-tagitem(A) ::= NOW(X).          { X.type = TSDB_DATA_TYPE_TIMESTAMP; tVariantCreate(&A, &X);}
+tagitem(A) ::= NOW(X).          { X.type = TSDB_DATA_TYPE_TIMESTAMP; tVariantCreateExt(&A, &X, TK_NOW);}
+
+tagitem(A) ::= NOW PLUS VARIABLE(X).{
+    X.type = TSDB_DATA_TYPE_TIMESTAMP;
+    tVariantCreateExt(&A, &X, TK_PLUS);
+}
+
+tagitem(A) ::= NOW MINUS VARIABLE(X).{
+    X.type = TSDB_DATA_TYPE_TIMESTAMP;
+    tVariantCreateExt(&A, &X, TK_MINUS);
+}
 
 tagitem(A) ::= MINUS(X) INTEGER(Y).{
     X.n += Y.n;

src/query/src/qExecutor.c

@@ -2775,7 +2775,7 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
   if (pDataStatis == NULL || pQueryAttr->numOfFilterCols == 0) {
     return true;
   }
-  bool ret = true;
+  bool ret = false;
   for (int32_t k = 0; k < pQueryAttr->numOfFilterCols; ++k) {
     SSingleColumnFilterInfo *pFilterInfo = &pQueryAttr->pFilterInfo[k];
 
@@ -2819,27 +2819,27 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
 
       for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
         if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
-           continue;
+           return true; //statis filter was not fit for in operator
         }
-        ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT);
-        if (ret == false) {
-          return false;
+        ret = pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT);
+        if (ret) { 
+          return true; //statis filter is coarse, so must return true if anyone is true 
         }
       }
     } else {
       for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
         if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
-           continue;
+          return true; //statis filter was not fit for in operator
         }
-        ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type);
-        if (ret == false) {
-          return false;
+        ret = pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type);
+        if (ret) {
+          return true; //statis filter is coarse, so must return true if anyone is true 
         }
       }
     }
   }
 
-  return ret;
+  return false;
 }
 
 static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) {

src/util/src/tconfig.c

@@ -467,7 +467,7 @@ bool taosReadGlobalCfg() {
 }
 
 void taosPrintGlobalCfg() {
-  uInfo("   taos config & system info:");
+  uInfo("        config & system info:");
   uInfo("==================================");
 
   for (int i = 0; i < tsGlobalConfigNum; ++i) {
@@ -553,7 +553,7 @@ static void taosDumpCfg(SGlobalCfg *cfg) {
 }
 
 void taosDumpGlobalCfg() {
-  printf("taos global config:\n");
+  printf("         global config:\n");
   printf("==================================\n");
   for (int i = 0; i < tsGlobalConfigNum; ++i) {
     SGlobalCfg *cfg = tsGlobalConfig + i;
@@ -564,7 +564,7 @@ void taosDumpGlobalCfg() {
     taosDumpCfg(cfg);
   }
 
-  printf("\ntaos local config:\n");
+  printf("\n      local config:\n");
   printf("==================================\n");
 
   for (int i = 0; i < tsGlobalConfigNum; ++i) {

tests/pytest/fulltest.sh

@@ -402,6 +402,7 @@ python3 ./test.py -f query/queryWildcardLength.py
 python3 ./test.py -f query/queryTbnameUpperLower.py
 python3 ./test.py -f query/queryGroupTbname.py
 python3 ./test.py -f insert/verifyMemToDiskCrash.py
+python3 ./test.py -f functions/variable_httpDbNameMandatory.py
 
 #======================p4-end===============
 

tests/pytest/functions/variable_httpDbNameMandatory.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import subprocess
+import random
+import math
+import numpy as np
+import inspect
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+from requests.auth import HTTPBasicAuth
+import requests
+import json
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to execute {__file__}")
+        tdSql.init(conn.cursor(), logSql)
+
+    def getBuildPath(self) -> str:
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/debug/build/bin")]
+                    break
+        return buildPath
+
+    def getCfgDir(self) -> str:
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            cfgDir = self.getBuildPath() + "/community/sim/dnode1/cfg"
+        else:
+            cfgDir = self.getBuildPath() + "/sim/dnode1/cfg"
+        return cfgDir
+
+    def getCfgFile(self) -> str:
+        return self.getCfgDir()+"/taos.cfg"
+
+    def rest_query(self,sql,db=''):                                       
+        host = '127.0.0.1'
+        user = 'root'
+        password = 'taosdata'
+        port =6041
+        if db == '':
+            url = "http://{}:{}/rest/sql".format(host, port )
+        else:
+            url = "http://{}:{}/rest/sql/{}".format(host, port, db )
+        try:
+            r = requests.post(url, 
+                data = 'use db' ,
+                auth = HTTPBasicAuth('root', 'taosdata'))  
+            r = requests.post(url, 
+                data = sql,
+                auth = HTTPBasicAuth('root', 'taosdata'))         
+        except:
+            print("REST API Failure (TODO: more info here)")
+            raise
+        rj = dict(r.json()['data'])
+        return rj
+    
+    def TS834(self):
+        tdLog.printNoPrefix("==========TS-782==========")
+        tdSql.prepare()
+
+        cfgfile = self.getCfgFile()
+
+        tdSql.execute("show variables")
+        res_com = tdSql.cursor.fetchall()
+        rescomlist = np.array(res_com)
+        cpms_index = np.where(rescomlist == "httpDbNameMandatory")
+        index_value = np.dstack((cpms_index[0])).squeeze()
+
+        tdSql.query("show variables")
+        tdSql.checkData(index_value, 1, 0)
+
+        rj = self.rest_query("show variables")
+        if 'httpDbNameMandatory' not in rj:
+            tdLog.info('has no httpDbNameMandatory shown')
+            tdLog.exit(1)
+        if rj['httpDbNameMandatory'] != '0':
+            tdLog.info('httpDbNameMandatory data:%s == expect:0'%rj['httpDbNameMandatory'])
+            tdLog.exit(1)
+        tdLog.info("httpDbNameMandatory by restful query data:%s == expect:0" % (rj['httpDbNameMandatory']))
+
+        tdSql.query("show dnodes")
+        index = tdSql.getData(0, 0)
+        tdLog.info("restart taosd ")
+        tdDnodes.stop(index)
+        cmd = f"echo 'httpDbNameMandatory 1' >> {cfgfile} "
+        try:
+            _ = subprocess.check_output(cmd, shell=True).decode("utf-8")
+        except Exception as e:
+            raise e
+
+        tdDnodes.start(index)
+        tdSql.query("show variables")
+        tdSql.checkData(index_value, 1, 1)
+
+        rj = self.rest_query("show variables", 'db')
+        if 'httpDbNameMandatory' not in rj:
+            tdLog.info('has no httpDbNameMandatory shown')
+            tdLog.exit(1)
+        if rj['httpDbNameMandatory'] != '1':
+            tdLog.info('httpDbNameMandatory data:%s == expect:0'%rj['httpDbNameMandatory'])
+            tdLog.exit(1)
+        tdLog.info("httpDbNameMandatory by restful query data:%s == expect:1" % (rj['httpDbNameMandatory']))
+
+    def run(self):
+        
+        #TS-834 https://jira.taosdata.com:18080/browse/TS-834
+        self.TS834()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
+
+
+

tests/script/api/batchprepare.c

@@ -2020,7 +2020,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) {
     code = taos_stmt_set_tbname_tags(stmt, buf, NULL);
     if (code != 0){
       printf("failed to execute taos_stmt_set_tbname_tags. code:%s\n", taos_stmt_errstr(stmt));
-      return -1;
+      goto exit;
     }  
 
     taos_stmt_bind_param_batch(stmt, params + id * 10);
@@ -2037,6 +2037,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) {
   unsigned long long endtime = getCurrentTime();
   printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10));
 
+exit:
   free(v.ts);  
   free(lb);
   free(params);
@@ -2227,7 +2228,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) {
   int code = taos_stmt_prepare(stmt, sql, 0);
   if (code != 0){
     printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt));
-    return -1;
+    goto exit;
     //exit(1);
   }
 
@@ -2255,6 +2256,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) {
   unsigned long long endtime = getCurrentTime();
   printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10));
 
+exit:
   free(v.ts);  
   free(lb);
   free(params);
@@ -2464,7 +2466,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) {
     code = taos_stmt_bind_param_batch(stmt, params + id * 10);
     if (code != 0) {
       printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt));
-      return -1;
+      goto exit;
     }
     
     taos_stmt_add_batch(stmt);
@@ -2480,6 +2482,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) {
   unsigned long long endtime = getCurrentTime();
   printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10));
 
+exit:
   free(v.ts);  
   free(lb);
   free(params);
@@ -2691,7 +2694,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) {
     code = taos_stmt_bind_param_batch(stmt, params + id * 10);
     if (code != 0) {
       printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt));
-      return -1;
+      goto exit;
     }
     
     taos_stmt_add_batch(stmt);
@@ -2707,6 +2710,8 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) {
   unsigned long long endtime = getCurrentTime();
   printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10));
 
+
+exit:
   free(v.ts);  
   free(lb);
   free(params);
@@ -2718,6 +2723,38 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) {
 }
 
 
+int stmt_funcb_autoctb_e6(TAOS_STMT *stmt) {
+  char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(now,?,?,?,?,?,?,?,?,?)";
+  int code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt));
+  }
+
+  return 0;
+}
+
+
+int stmt_funcb_autoctb_e7(TAOS_STMT *stmt) {
+  char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,true,?,?,?,?,?,?,?,?)";
+  int code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt));
+  }
+
+  return 0;
+}
+
+
+int stmt_funcb_autoctb_e8(TAOS_STMT *stmt) {
+  char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,1,?,?,?,?,?,?,?)";
+  int code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt));
+  }
+
+  return 0;
+}
+
 
 //300 tables 60 records
 int stmt_funcb1(TAOS_STMT *stmt) {
@@ -4853,6 +4890,44 @@ void* runcase(void *par) {
 
 #endif
 
+
+#if 1  
+  prepare(taos, 1, 0);
+
+  stmt = taos_stmt_init(taos);
+
+  printf("e6 start\n");
+  stmt_funcb_autoctb_e6(stmt);
+  printf("e6 end\n");
+  taos_stmt_close(stmt);
+
+#endif
+
+#if 1  
+  prepare(taos, 1, 0);
+
+  stmt = taos_stmt_init(taos);
+
+  printf("e7 start\n");
+  stmt_funcb_autoctb_e7(stmt);
+  printf("e7 end\n");
+  taos_stmt_close(stmt);
+
+#endif
+
+#if 1  
+  prepare(taos, 1, 0);
+
+  stmt = taos_stmt_init(taos);
+
+  printf("e8 start\n");
+  stmt_funcb_autoctb_e8(stmt);
+  printf("e8 end\n");
+  taos_stmt_close(stmt);
+  
+#endif
+
+
 #if 1
   prepare(taos, 1, 0);
 

tests/script/fullGeneralSuite.sim

@@ -223,3 +223,4 @@ run general/db/show_create_db.sim
 run general/db/show_create_table.sim
 run general/parser/like.sim
 run general/parser/interp_blocks.sim
+run general/parser/create_tb_with_timestamp_tag.sim

tests/script/general/parser/create_tb_with_timestamp_tag.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2
+system sh/exec.sh -n dnode1 -s start
+
+sleep 100
+sql connect
+print ======================== dnode1 start
+
+$db = testdb
+
+sql create database $db precision 'ns'
+sql use $db
+
+sql create stable st1 (ts timestamp , c1 int) tags(t1 timestamp, t2 int)
+
+sql create table t1_0 using st1 tags(now, 0)
+#nanoseconds
+sql create table t1_1 using st1 tags(now + 1b, 0)
+#microseconds
+sql create table t1_2 using st1 tags(now + 1u, 0)
+#milliseconds
+sql create table t1_3 using st1 tags(now + 1a, 0)
+#seconds
+sql create table t1_4 using st1 tags(now + 1s, 0)
+#minutes
+sql create table t1_5 using st1 tags(now + 1m, 0)
+#hours
+sql create table t1_6 using st1 tags(now + 1h, 0)
+#days
+sql create table t1_7 using st1 tags(now + 1d, 0)
+#weeks
+sql create table t1_8 using st1 tags(now + 1w, 0)
+#months(not supported)
+sql_error create table t1_9 using st1 tags(now + 1n, 0)
+#years(not supported)
+sql_error create table t1_10 using st1 tags(now + 1y, 0)
+
+
+sql create stable st2 (ts timestamp , c1 int) tags(t1 timestamp, t2 int)
+
+sql create table t2_0 using st2 tags(now, 0)
+#nanoseconds
+sql create table t2_1 using st2 tags(now - 1b, 0)
+#microseconds
+sql create table t2_2 using st2 tags(now - 1u, 0)
+#milliseconds
+sql create table t2_3 using st2 tags(now - 1a, 0)
+#seconds
+sql create table t2_4 using st2 tags(now - 1s, 0)
+#minutes
+sql create table t2_5 using st2 tags(now - 1m, 0)
+#hours
+sql create table t2_6 using st2 tags(now - 1h, 0)
+#days
+sql create table t2_7 using st2 tags(now - 1d, 0)
+#weeks
+sql create table t2_8 using st2 tags(now - 1w, 0)
+#months(not supported)
+sql_error create table t2_9 using st2 tags(now - 1n, 0)
+#years(not supported)
+sql_error create table t2_10 using st2 tags(now - 1y, 0)
+
+sql insert into t1_0 values (now, 0)
+sql insert into t1_1 values (now, 1)
+sql insert into t1_2 values (now, 2)
+sql insert into t1_3 values (now, 3)
+sql insert into t1_4 values (now, 4)
+sql insert into t1_5 values (now, 5)
+sql insert into t1_6 values (now, 6)
+sql insert into t1_7 values (now, 7)
+sql insert into t1_8 values (now, 8)
+
+sql insert into t2_0 values (now, 0)
+sql insert into t2_1 values (now, 1)
+sql insert into t2_2 values (now, 2)
+sql insert into t2_3 values (now, 3)
+sql insert into t2_4 values (now, 4)
+sql insert into t2_5 values (now, 5)
+sql insert into t2_6 values (now, 6)
+sql insert into t2_7 values (now, 7)
+sql insert into t2_8 values (now, 8)
+
+sql select * from st1
+
+if $rows != 9 then
+  return -1
+endi
+
+sql select * from st2
+
+if $rows != 9 then
+  return -1
+endi
+
+sql create stable st3 (ts timestamp , c1 int) tags (t1 timestamp, t2 timestamp, t3 timestamp, t4 timestamp, t5 timestamp, t6 timestamp, t7 timestamp, t8 timestamp, t9 timestamp)
+sql create table t3 using st3 tags(now, now + 1b, now + 1u, now + 1a, now + 1s, now + 1m, now + 1h, now + 1d, now + 1w)
+sql insert into t3 values (now, 1)
+
+sql select * from st3
+if $rows != 1 then
+  return -1
+endi
+
+sql create stable st4 (ts timestamp , c1 int) tags (t1 timestamp, t2 timestamp, t3 timestamp, t4 timestamp, t5 timestamp, t6 timestamp, t7 timestamp, t8 timestamp, t9 timestamp)
+sql create table t4 using st4 tags(now, now - 1b, now - 1u, now - 1a, now - 1s, now - 1m, now - 1h, now - 1d, now - 1w)
+sql insert into t4 values (now, 1)
+
+sql select * from st4
+if $rows != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT