Merge branch '3.0' of github.com:taosdata/TDengine into 3.0

include/libs/scalar/scalar.h

@@ -111,6 +111,8 @@ int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar
 int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 #ifdef __cplusplus
 }

source/client/src/clientMain.c

@@ -142,6 +142,7 @@ void taos_close(TAOS *taos) {
 
 int taos_errno(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
+    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     return terrno;
   }
 
@@ -149,11 +150,13 @@ int taos_errno(TAOS_RES *res) {
     return 0;
   }
 
-  return ((SRequestObj *)res)->code;
+  return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL
+                                                              : ((SRequestObj *)res)->code;
 }
 
 const char *taos_errstr(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
+    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     return (const char *)tstrerror(terrno);
   }
 
@@ -165,7 +168,8 @@ const char *taos_errstr(TAOS_RES *res) {
   if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
     return pRequest->msgBuf;
   } else {
-    return (const char *)tstrerror(pRequest->code);
+    return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL)
+                                                    : (const char *)tstrerror(pRequest->code);
   }
 }
 

source/libs/executor/src/scanoperator.c

@@ -521,8 +521,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
 
     int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader,
                                   GET_TASKID(pTaskInfo));
-    if (code != 0) {
-      // TODO
+    if (code != TSDB_CODE_SUCCESS) {
+      longjmp(pTaskInfo->env, code);
+      return NULL;
     }
   }
 

source/libs/function/src/builtins.c

@@ -2369,6 +2369,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getHLLFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = hllFunction,
+    .sprocessFunc = hllScalarFunction,
     .finalizeFunc = hllFinalize,
     .invertFunc   = NULL,
     .combineFunc  = hllCombine,
@@ -2437,6 +2438,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getCsumFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = csumFunction,
+    .sprocessFunc = csumScalarFunction,
     .finalizeFunc = NULL
   },
   {
@@ -2463,7 +2465,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "tail",
     .type = FUNCTION_TYPE_TAIL,
-    .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | 
+    .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC |
                       FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateTail,
     .getEnvFunc   = getTailFuncEnv,
@@ -2474,7 +2476,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "unique",
     .type = FUNCTION_TYPE_UNIQUE,
-    .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | 
+    .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC |
                       FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateUnique,
     .getEnvFunc   = getUniqueFuncEnv,

source/libs/function/src/builtinsimpl.c

@@ -2723,6 +2723,9 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
 
   int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
 
+  //  please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function.
+  //  we will use this opt implementation in an new version that is only available in scan subplan
+#if 0
   if (blockDataOrder == TSDB_ORDER_ASC) {
     // filter according to current result firstly
     if (pResInfo->numOfRes > 0) {
@@ -2770,6 +2773,22 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
       }
     }
   }
+#else
+  for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
+    if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
+      continue;
+    }
+
+    numOfElems++;
+
+    char* data = colDataGetData(pInputCol, i);
+    TSKEY cts = getRowPTs(pInput->pPTS, i);
+    if (pResInfo->numOfRes == 0 || pInfo->ts > cts) {
+      doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data);
+      pResInfo->numOfRes = 1;
+    }
+  }
+#endif
 
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;
@@ -2801,6 +2820,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
 
   int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
 
+  //  please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function.
+#if 0
   if (blockDataOrder == TSDB_ORDER_ASC) {
     for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
       if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
@@ -2833,6 +2854,22 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
       break;
     }
   }
+#else
+  for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+    if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
+      continue;
+    }
+
+    numOfElems++;
+
+    char* data = colDataGetData(pInputCol, i);
+    TSKEY cts = getRowPTs(pInput->pPTS, i);
+    if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
+      doSaveCurrentVal(pCtx, i, cts, type, data);
+      pResInfo->numOfRes = 1;
+    }
+  }
+#endif
 
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;
@@ -2988,6 +3025,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
 
   int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
 
+#if 0
+  // the optimized version only function if all tuples in one block are monotonious increasing or descreasing.
+  // this is NOT always works if project operator exists in downstream.
   if (blockDataOrder == TSDB_ORDER_ASC) {
     for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
       char* data = colDataGetData(pInputCol, i);
@@ -2997,6 +3037,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
       if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
         doSaveLastrow(pCtx, data, i, cts, pInfo);
       }
+
       break;
     }
   } else {  // descending order
@@ -3011,7 +3052,19 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
       break;
     }
   }
+#else
+  for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+    char* data = colDataGetData(pInputCol, i);
+    TSKEY cts = getRowPTs(pInput->pPTS, i);
+    numOfElems++;
+
+    if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
+      doSaveLastrow(pCtx, data, i, cts, pInfo);
+      pResInfo->numOfRes = 1;
+    }
+  }
 
+#endif
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;
 }
@@ -5926,6 +5979,7 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) {
     TSKEY cts = getRowPTs(pInput->pPTS, i);
     if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
       doSaveLastrow(pCtx, data, i, cts, pInfo);
+      pResInfo->numOfRes = 1;
     }
   }
 

source/libs/scalar/src/sclfunc.c

@@ -1746,20 +1746,14 @@ int32_t countScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
   SColumnInfoData *pOutputData = pOutput->columnData;
 
   int64_t *out = (int64_t *)pOutputData->pData;
-  bool hasNull = false;
   *out = 0;
   for (int32_t i = 0; i < pInput->numOfRows; ++i) {
     if (colDataIsNull_s(pInputData, i)) {
-      hasNull = true;
-      break;
+      continue;
     }
     (*out)++;
   }
 
-  if (hasNull) {
-    colDataAppendNULL(pOutputData, 0);
-  }
-
   pOutput->numOfRows = 1;
   return TSDB_CODE_SUCCESS;
 }
@@ -2421,3 +2415,11 @@ int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *
 int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   return avgScalarFunction(pInput, inputNum, pOutput);
 }
+
+int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return countScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return sumScalarFunction(pInput, inputNum, pOutput);
+}

tests/script/tsim/valgrind/basic3.sim

 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/deploy.sh -n dnode2 -i 2
+system sh/cfg.sh -n dnode1 -c debugflag -v 131
 system sh/exec.sh -n dnode1 -s start -v
-system sh/exec.sh -n dnode2 -s start -v
 sql connect
 
-print =============== add dnode2 into cluster
-sql create dnode $hostname port 7200
-
+print =============== step1: create drop show dnodes
 $x = 0
 step1:
 	$x = $x + 1
@@ -18,135 +15,59 @@ step1:
 	endi
 sql show dnodes
 print ---> $data00 $data01 $data02 $data03 $data04 $data05
-print ---> $data10 $data11 $data12 $data13 $data14 $data15
-if $rows != 2 then
+if $rows != 1 then
   return -1
 endi
 if $data(1)[4] != ready then
   goto step1
 endi
-if $data(2)[4] != ready then
-  goto step1
-endi
-
-print =============== create database, stable, table
-sql create database db vgroups 3
-sql use db
-sql create table stb (ts timestamp, c int) tags (t int)
-sql create table t0 using stb tags (0)
-sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10));
-
-print =============== run show xxxx
-sql show dnodes
-if $rows != 2 then 
-  return -1
-endi
-
-sql show mnodes
-if $rows != 1 then 
-  return -1
-endi
 
+print =============== step2: create db
+sql create database d1 vgroups 2 buffer 3
 sql show databases
-if $rows != 3 then
-  return -1
-endi
+sql use d1
+sql show vgroups
 
+print =============== step3: create show stable
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
 sql show stables
 if $rows != 1 then 
   return -1
 endi
 
+print =============== step4: create show table
+sql create table ct1 using stb tags(1000)
+sql create table ct2 using stb tags(2000)
+sql create table ct3 using stb tags(3000)
 sql show tables
-if $rows != 2 then 
-  return -1
-endi
-
-sql show users
-if $rows != 1 then 
-  return -1
-endi
-
-sql show vgroups
-if $rows != 3 then 
-  return -1
-endi
-
-print =============== run select * from information_schema.xxxx
-sql select * from information_schema.`dnodes`
-if $rows != 2 then 
-  return -1
-endi
-
-sql select * from information_schema.`mnodes`
-if $rows != 1 then 
-  return -1
-endi
-
-sql select * from information_schema.user_databases
-if $rows != 3 then
-  return -1
-endi
-
-sql select * from information_schema.user_stables
-if $rows != 1 then 
-  return -1
-endi
-
-sql select * from information_schema.user_tables
-if $rows != 31 then
-  return -1
-endi
-
-sql select * from information_schema.user_users
-if $rows != 1 then 
-  return -1
-endi
-
-sql select * from information_schema.`vgroups`
 if $rows != 3 then 
   return -1
 endi
 
-sql show variables;
-if $rows != 4 then
-  return -1
-endi
+print =============== step5: insert data
+sql insert into ct1 values(now+0d, 10, 2.0, 3.0)
+sql insert into ct1 values(now+1d, 11, 2.1, 3.1)(now+2d, -12, -2.2, -3.2)(now+3d, -13, -2.3, -3.3)
+sql insert into ct2 values(now+0d, 10, 2.0, 3.0)
+sql insert into ct2 values(now+1d, 11, 2.1, 3.1)(now+2d, -12, -2.2, -3.2)(now+3d, -13, -2.3, -3.3)
+sql insert into ct3 values('2022-01-01 00:00:00.000', 10, 2.0, 3.0)
 
-sql show dnode 1 variables;
-if $rows <= 0 then
-  return -1
-endi
-
-sql show local variables;
-if $rows <= 0 then
-  return -1
-endi
+print =============== step6: query data
+sql select * from ct1 where ts < now -1d and ts > now +1d 
+sql select * from stb where ts < now -1d and ts > now +1d 
+sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc 
+sql select * from stb where ts < now -1d and ts > now +1d order by ts desc 
 
-print ==== stop dnode1 and dnode2, and restart dnodes
+_OVER:
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
-
-print =============== check dnode1
-system_content sh/checkValgrind.sh -n dnode1 
-print cmd return result ----> [ $system_content ]
-if $system_content <= 0 then
-  return 0
-endi 
-
+print =============== check
 $null=
-if $system_content == $null then
-  return 0
-endi 
 
-print =============== check dnode2
-system_content sh/checkValgrind.sh -n dnode2 
+system_content sh/checkValgrind.sh -n dnode1 
 print cmd return result ----> [ $system_content ]
-if $system_content <= 0 then
-  return 0
+if $system_content > 1 then
+  return -1
 endi 
 
-$null=
 if $system_content == $null then
-  return 0
+  return -1
 endi 

tests/script/tsim/valgrind/checkError2.sim

@@ -23,7 +23,7 @@ if $data(1)[4] != ready then
 endi
 
 print =============== step2: create db
-sql create database d1 vgroups 1 buffer 3
+sql create database d1 vgroups 2 buffer 3
 sql show databases
 sql use d1
 sql show vgroups
@@ -44,24 +44,32 @@ if $rows != 3 then
   return -1
 endi
 
-print =============== step5: insert data
+print =============== step5: insert data (null / update)
 sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
-sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
+sql insert into ct1 values(now+1s, 11, 2.1, NULL)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
 sql insert into ct2 values(now+0s, 10, 2.0, 3.0)
 sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
-sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0)
+sql insert into ct3 values('2021-01-01 00:00:00.000', NULL, NULL, 3.0)
+sql insert into ct3 values('2022-03-02 16:59:00.010', 3  , 4, 5), ('2022-03-02 16:59:00.010', 33 , 4, 5), ('2022-04-01 16:59:00.011', 4,  4, 5), ('2022-04-01 16:59:00.011', 6,  4, 5), ('2022-03-06 16:59:00.013', 8,  4, 5);
+sql insert into ct3 values('2022-03-02 16:59:00.010', 103, 1, 2), ('2022-03-02 16:59:00.010', 303, 3, 4), ('2022-04-01 16:59:00.011', 40, 5, 6), ('2022-04-01 16:59:00.011', 60, 4, 5), ('2022-03-06 16:59:00.013', 80, 4, 5);
 
 print =============== step6: query data
 sql select * from ct1
 sql select * from stb
 sql select c1, c2, c3 from ct1
 sql select ts, c1, c2, c3 from stb
+sql select * from ct1 where ts < now -1d and ts > now +1d 
+sql select * from stb where ts < now -1d and ts > now +1d 
+#sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc 
+#sql select * from stb where ts < now -1d and ts > now +1d order by ts desc 
 
 print =============== step7: count
 sql select count(*) from ct1;
 sql select count(*) from stb;
 sql select count(ts), count(c1), count(c2), count(c3) from ct1
 sql select count(ts), count(c1), count(c2), count(c3) from stb
+sql select count(*) from ct1 where ts < now -1d and ts > now +1d 
+sql select count(*) from stb where ts < now -1d and ts > now +1d
 
 print =============== step8: func
 sql select first(ts), first(c1), first(c2), first(c3) from ct1
@@ -69,6 +77,20 @@ sql select min(c1), min(c2), min(c3) from ct1
 sql select max(c1), max(c2), max(c3) from ct1
 sql select sum(c1), sum(c2), sum(c3) from ct1
 
+print =============== step9: insert select
+#sql create table ct4 using stb tags(4000);
+#sql insert into ct4 select * from ct1;
+#sql select * from ct4;
+#sql insert into ct4 select ts,c1,c2,c3 from stb;
+
+#sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double);
+#sql insert into tb1 (ts, c1, c2, c3) select * from ct1;
+#sql select * from tb1;
+
+#sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double);
+#sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2;
+#sql select * from tb2;
+
 _OVER:
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 print =============== check

taos-tools @ bd496f76

-Subproject commit d807c3ffa6f750f7765e102917d1328cadf21c13
+Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde

taosws-rs @ 7a94ffab

-Subproject commit c5fded266d3b10508e38bf3285bb7ecf798bc343
+Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6