Merge branch 'master' into hotfix/TS-854

Jenkinsfile

@@ -38,7 +38,8 @@ def pre_test(){
     sudo rmtaos || echo "taosd has not installed"
     '''
     sh '''
-    killall -9 taosd ||echo "no taosd running"
+    kill -9 $(pidof taosd) ||echo "no taosd running"
+    kill -9 $(pidof taosadapter) ||echo "no taosadapter running"
     killall -9 gdb || echo "no gdb running"
     killall -9 python3.8 || echo "no python program running"
     cd ${WKC}

src/client/src/tscPrepare.c

@@ -1527,6 +1527,41 @@ int stmtGenInsertStatement(SSqlObj* pSql, STscStmt* pStmt, const char* name, TAO
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t stmtValidateValuesFields(SSqlCmd *pCmd, char * sql) {
+  int32_t loopCont = 1, index0 = 0, values = 0;
+  SStrToken sToken;
+  
+  while (loopCont) {
+    sToken = tStrGetToken(sql, &index0, false);
+    if (sToken.n <= 0) {
+      return TSDB_CODE_SUCCESS;
+    }
+  
+    switch (sToken.type) {
+      case TK_RP:
+        if (values) {
+         return TSDB_CODE_SUCCESS;
+        }
+        break;
+      case TK_VALUES:
+        values = 1;
+        break;
+      case TK_QUESTION:  
+      case TK_LP:
+        break;
+      default:
+        if (values) {
+          tscError("only ? allowed in values");
+          return tscInvalidOperationMsg(pCmd->payload, "only ? allowed in values", NULL);
+        }
+        break;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 ////////////////////////////////////////////////////////////////////////////////
 // interface functions
 
@@ -1631,6 +1666,11 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
       STMT_RET(ret);
     }
 
+    ret = stmtValidateValuesFields(&pSql->cmd, pSql->sqlstr);
+    if (ret != TSDB_CODE_SUCCESS) {
+      STMT_RET(ret);
+    }
+
     if (pStmt->multiTbInsert) {
       STMT_RET(TSDB_CODE_SUCCESS);
     }

src/client/src/tscSQLParser.c

@@ -7264,7 +7264,7 @@ int32_t validateFunctionFromUpstream(SQueryInfo* pQueryInfo, char* msg) {
         SQueryInfo* pUp = taosArrayGetP(pQueryInfo->pUpstream, j);
         STableMetaInfo  *pTableMetaInfo = tscGetMetaInfo(pUp, 0);
         bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
-        if ((!isSTable) || groupbyTbname(pUp)) {
+        if ((!isSTable) || groupbyTbname(pUp) || pUp->interval.interval > 0) {
           return TSDB_CODE_SUCCESS;
         }
       }

src/common/src/tglobal.c

@@ -1279,7 +1279,7 @@ static void doInitGlobalConfig(void) {
   cfg.option = "httpDbNameMandatory";
   cfg.ptr = &tsHttpDbNameMandatory;
   cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = 0;
   cfg.maxValue = 1;
   cfg.ptrLength = 0;

src/kit/shell/src/shellCheck.c

@@ -36,6 +36,7 @@ typedef struct {
   int       totalThreads;
   void *    taos;
   char *    db;
+  int       code;
 } ShellThreadObj;
 
 static int32_t shellUseDb(TAOS *con, char *db) {
@@ -112,10 +113,10 @@ static void *shellCheckThreadFp(void *arg) {
   int32_t end = (pThread->threadIndex + 1) * interval;
 
   if (start >= tbNum) return NULL;
-  if (end > tbNum) end = tbNum + 1;
+  if (end > tbNum) end = tbNum;
 
   char file[32] = {0};
-  snprintf(file, 32, "tb%d.txt", pThread->threadIndex);
+  snprintf(file, 32, "tb%d.sql", pThread->threadIndex);
 
   FILE *fp = fopen(file, "w");
   if (!fp) {
@@ -123,16 +124,19 @@ static void *shellCheckThreadFp(void *arg) {
     return NULL;
   }
 
+  ASSERT(pThread->code != 0);
+
   char sql[SHELL_SQL_LEN];
   for (int32_t t = start; t < end; ++t) {
     char *tbname = tbNames[t];
     if (tbname == NULL) break;
 
-    snprintf(sql, SHELL_SQL_LEN, "select * from %s limit 1", tbname);
+    snprintf(sql, SHELL_SQL_LEN, "select last_row(_c0) from %s;", tbname);
 
     TAOS_RES *pSql = taos_query(pThread->taos, sql);
     int32_t   code = taos_errno(pSql);
-    if (code != 0) {
+    // -k: -1 means check all errors, while other non-zero values means check specific errors.
+    if ((code == pThread->code) || ((pThread->code == -1) && (code != 0))) {
       int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname);
       fwrite(sql, 1, len, fp);
       atomic_add_fetch_32(&errorNum, 1);
@@ -161,6 +165,7 @@ static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) {
     pThread->totalThreads = _args->threadNum;
     pThread->taos = con;
     pThread->db = _args->database;
+    pThread->code = _args->check;
 
     pthread_attr_init(&thattr);
     pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);

src/kit/taosdemo/taosdemo.c

@@ -10963,8 +10963,6 @@ static int insertTestProcess() {
     assert(cmdBuffer);
 
     if(createDatabasesAndStables(cmdBuffer) != 0) {
-        if (g_fpOfInsertResult)
-            fclose(g_fpOfInsertResult);
         free(cmdBuffer);
         return -1;
     }

src/sync/src/syncMain.c

@@ -1502,7 +1502,10 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
       }
     }
 
-    int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen);
+    SOCKET peerFd = pPeer->peerFd;
+    pthread_mutex_unlock(&pNode->mutex);
+    int32_t retLen = taosWriteMsg(peerFd, pSyncHead, fwdLen);
+    pthread_mutex_lock(&pNode->mutex);
     if (retLen == fwdLen) {
       sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, syncRole[pPeer->role],
              syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len);

src/tsdb/src/tsdbMain.c

@@ -505,6 +505,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   }
 
   // Check keep
+#if 0  // already checked and set in mnodeSetDefaultDbCfg
   if (pCfg->keep == -1) {
     pCfg->keep = TSDB_DEFAULT_KEEP;
   } else {
@@ -525,6 +526,25 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   if (pCfg->keep2 == 0) {
     pCfg->keep2 = pCfg->keep;
   }
+#endif
+
+  int32_t keepMin = pCfg->keep1;
+  int32_t keepMid = pCfg->keep2;
+  int32_t keepMax = pCfg->keep;
+
+  if (keepMin > keepMid) {
+    SWAP(keepMin, keepMid, int32_t);
+  }
+  if (keepMin > keepMax) {
+    SWAP(keepMin, keepMax, int32_t);
+  }
+  if (keepMid > keepMax) {
+    SWAP(keepMid, keepMax, int32_t);
+  }
+
+  pCfg->keep = keepMax;
+  pCfg->keep1 = keepMin;
+  pCfg->keep2 = keepMid;
 
   // update check
   if (pCfg->update < TD_ROW_DISCARD_UPDATE || pCfg->update > TD_ROW_PARTIAL_UPDATE)

tests/pytest/fulltest.sh

@@ -402,6 +402,7 @@ python3 ./test.py -f query/queryWildcardLength.py
 python3 ./test.py -f query/queryTbnameUpperLower.py
 python3 ./test.py -f query/queryGroupTbname.py
 python3 ./test.py -f insert/verifyMemToDiskCrash.py
+python3 ./test.py -f functions/variable_httpDbNameMandatory.py
 
 #======================p4-end===============
 

tests/pytest/functions/variable_httpDbNameMandatory.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import subprocess
+import random
+import math
+import numpy as np
+import inspect
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+from requests.auth import HTTPBasicAuth
+import requests
+import json
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to execute {__file__}")
+        tdSql.init(conn.cursor(), logSql)
+
+    def getBuildPath(self) -> str:
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/debug/build/bin")]
+                    break
+        return buildPath
+
+    def getCfgDir(self) -> str:
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            cfgDir = self.getBuildPath() + "/community/sim/dnode1/cfg"
+        else:
+            cfgDir = self.getBuildPath() + "/sim/dnode1/cfg"
+        return cfgDir
+
+    def getCfgFile(self) -> str:
+        return self.getCfgDir()+"/taos.cfg"
+
+    def rest_query(self,sql,db=''):                                       
+        host = '127.0.0.1'
+        user = 'root'
+        password = 'taosdata'
+        port =6041
+        if db == '':
+            url = "http://{}:{}/rest/sql".format(host, port )
+        else:
+            url = "http://{}:{}/rest/sql/{}".format(host, port, db )
+        try:
+            r = requests.post(url, 
+                data = 'use db' ,
+                auth = HTTPBasicAuth('root', 'taosdata'))  
+            r = requests.post(url, 
+                data = sql,
+                auth = HTTPBasicAuth('root', 'taosdata'))         
+        except:
+            print("REST API Failure (TODO: more info here)")
+            raise
+        rj = dict(r.json()['data'])
+        return rj
+    
+    def TS834(self):
+        tdLog.printNoPrefix("==========TS-782==========")
+        tdSql.prepare()
+
+        cfgfile = self.getCfgFile()
+
+        tdSql.execute("show variables")
+        res_com = tdSql.cursor.fetchall()
+        rescomlist = np.array(res_com)
+        cpms_index = np.where(rescomlist == "httpDbNameMandatory")
+        index_value = np.dstack((cpms_index[0])).squeeze()
+
+        tdSql.query("show variables")
+        tdSql.checkData(index_value, 1, 0)
+
+        rj = self.rest_query("show variables")
+        if 'httpDbNameMandatory' not in rj:
+            tdLog.info('has no httpDbNameMandatory shown')
+            tdLog.exit(1)
+        if rj['httpDbNameMandatory'] != '0':
+            tdLog.info('httpDbNameMandatory data:%s == expect:0'%rj['httpDbNameMandatory'])
+            tdLog.exit(1)
+        tdLog.info("httpDbNameMandatory by restful query data:%s == expect:0" % (rj['httpDbNameMandatory']))
+
+        tdSql.query("show dnodes")
+        index = tdSql.getData(0, 0)
+        tdLog.info("restart taosd ")
+        tdDnodes.stop(index)
+        cmd = f"echo 'httpDbNameMandatory 1' >> {cfgfile} "
+        try:
+            _ = subprocess.check_output(cmd, shell=True).decode("utf-8")
+        except Exception as e:
+            raise e
+
+        tdDnodes.start(index)
+        tdSql.query("show variables")
+        tdSql.checkData(index_value, 1, 1)
+
+        rj = self.rest_query("show variables", 'db')
+        if 'httpDbNameMandatory' not in rj:
+            tdLog.info('has no httpDbNameMandatory shown')
+            tdLog.exit(1)
+        if rj['httpDbNameMandatory'] != '1':
+            tdLog.info('httpDbNameMandatory data:%s == expect:0'%rj['httpDbNameMandatory'])
+            tdLog.exit(1)
+        tdLog.info("httpDbNameMandatory by restful query data:%s == expect:1" % (rj['httpDbNameMandatory']))
+
+    def run(self):
+        
+        #TS-834 https://jira.taosdata.com:18080/browse/TS-834
+        self.TS834()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
+
+
+

tests/pytest/query/unionAllTest.py

@@ -116,10 +116,41 @@ class TDTestCase:
                 sql += f"union all select last(*) from sub{i} "
 
             tdSql.execute("create table sub%d using st2 tags('nchar%d')" % (i, i))
-            tdSql.execute("insert into sub%d values(%d, %d, %d, %d)" % (i, self.ts + i, i, i, i))
+            tdSql.execute("insert into sub%d values(%d, %d, %d, %d)(%d, %d, %d, %d)" % (i, self.ts + i, i, i, i,self.ts + i + 101, i + 101, i + 101, i + 101))
 
         tdSql.error(sql)
 
+        # TS-795
+        tdLog.info("test case for TS-795")
+        
+        functions = ["*", "count", "avg", "twa", "irate", "sum", "stddev", "leastsquares", "min", "max", "first", "last", "top", "bottom", "percentile", "apercentile", "last_row"]
+        
+        for func in functions:
+            expr = func
+            if func == "top" or func == "bottom":
+                expr += "(c1, 1)"
+            elif func == "percentile" or func == "apercentile":
+                expr += "(c1, 0.5)"
+            elif func == "leastsquares":
+                expr = func + "(c1, 1, 1)"
+            elif func == "*":
+                expr = func
+            else:
+                expr += "(c1)"
+
+            for i in range(100):
+                if i == 0:
+                    sql = f"select {expr} from sub0 "
+                else:
+                    sql += f"union all select {expr} from sub{i} "
+
+            tdSql.query(sql)
+            if func == "*":
+                tdSql.checkRows(200)
+            else:
+                tdSql.checkRows(100)
+
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/script/api/batchprepare.c

@@ -2020,7 +2020,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) {
     code = taos_stmt_set_tbname_tags(stmt, buf, NULL);
     if (code != 0){
       printf("failed to execute taos_stmt_set_tbname_tags. code:%s\n", taos_stmt_errstr(stmt));
-      return -1;
+      goto exit;
     }  
 
     taos_stmt_bind_param_batch(stmt, params + id * 10);
@@ -2037,6 +2037,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) {
   unsigned long long endtime = getCurrentTime();
   printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10));
 
+exit:
   free(v.ts);  
   free(lb);
   free(params);
@@ -2227,7 +2228,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) {
   int code = taos_stmt_prepare(stmt, sql, 0);
   if (code != 0){
     printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt));
-    return -1;
+    goto exit;
     //exit(1);
   }
 
@@ -2255,6 +2256,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) {
   unsigned long long endtime = getCurrentTime();
   printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10));
 
+exit:
   free(v.ts);  
   free(lb);
   free(params);
@@ -2464,7 +2466,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) {
     code = taos_stmt_bind_param_batch(stmt, params + id * 10);
     if (code != 0) {
       printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt));
-      return -1;
+      goto exit;
     }
     
     taos_stmt_add_batch(stmt);
@@ -2480,6 +2482,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) {
   unsigned long long endtime = getCurrentTime();
   printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10));
 
+exit:
   free(v.ts);  
   free(lb);
   free(params);
@@ -2691,7 +2694,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) {
     code = taos_stmt_bind_param_batch(stmt, params + id * 10);
     if (code != 0) {
       printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt));
-      return -1;
+      goto exit;
     }
     
     taos_stmt_add_batch(stmt);
@@ -2707,6 +2710,8 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) {
   unsigned long long endtime = getCurrentTime();
   printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10));
 
+
+exit:
   free(v.ts);  
   free(lb);
   free(params);
@@ -2718,6 +2723,38 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) {
 }
 
 
+int stmt_funcb_autoctb_e6(TAOS_STMT *stmt) {
+  char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(now,?,?,?,?,?,?,?,?,?)";
+  int code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt));
+  }
+
+  return 0;
+}
+
+
+int stmt_funcb_autoctb_e7(TAOS_STMT *stmt) {
+  char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,true,?,?,?,?,?,?,?,?)";
+  int code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt));
+  }
+
+  return 0;
+}
+
+
+int stmt_funcb_autoctb_e8(TAOS_STMT *stmt) {
+  char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,1,?,?,?,?,?,?,?)";
+  int code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt));
+  }
+
+  return 0;
+}
+
 
 //300 tables 60 records
 int stmt_funcb1(TAOS_STMT *stmt) {
@@ -4853,6 +4890,44 @@ void* runcase(void *par) {
 
 #endif
 
+
+#if 1  
+  prepare(taos, 1, 0);
+
+  stmt = taos_stmt_init(taos);
+
+  printf("e6 start\n");
+  stmt_funcb_autoctb_e6(stmt);
+  printf("e6 end\n");
+  taos_stmt_close(stmt);
+
+#endif
+
+#if 1  
+  prepare(taos, 1, 0);
+
+  stmt = taos_stmt_init(taos);
+
+  printf("e7 start\n");
+  stmt_funcb_autoctb_e7(stmt);
+  printf("e7 end\n");
+  taos_stmt_close(stmt);
+
+#endif
+
+#if 1  
+  prepare(taos, 1, 0);
+
+  stmt = taos_stmt_init(taos);
+
+  printf("e8 start\n");
+  stmt_funcb_autoctb_e8(stmt);
+  printf("e8 end\n");
+  taos_stmt_close(stmt);
+  
+#endif
+
+
 #if 1
   prepare(taos, 1, 0);
 

tests/script/general/parser/function.sim

@@ -1197,3 +1197,16 @@ if $data00 != 0.000000000 then
   return -1
 endi
 
+sql select derivative(aa,1m,0) from (select avg(voltage) as aa from smeters where ts>='2021-08-08 10:10:10.000' and ts < '2021-08-08 10:10:20.000'  and current=10 interval(1000a));
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-08-08 10:10:12.000@ then
+  return -1
+endi
+if $data01 != 0.000000000 then
+  return -1
+endi
+
+
+