Merge branch '3.0' into cpwu/3.0

include/common/ttime.h

@@ -64,6 +64,7 @@ char getPrecisionUnit(int32_t precision);
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
+int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal);
 
 void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
 

include/libs/function/functionMgt.h

@@ -85,8 +85,8 @@ typedef enum EFunctionType {
   // conversion function
   FUNCTION_TYPE_CAST = 2000,
   FUNCTION_TYPE_TO_ISO8601,
+  FUNCTION_TYPE_TO_UNIXTIMESTAMP,
   FUNCTION_TYPE_TO_JSON,
-  FUNCTION_TYPE_UNIXTIMESTAMP,
 
   // date and time function
   FUNCTION_TYPE_NOW = 2500,

include/libs/scalar/scalar.h

@@ -75,6 +75,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 
 /* Time related functions */
 int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 

packaging/install.sh

@@ -199,6 +199,8 @@ function install_lib() {
     ${csudo} rm -f ${lib_link_dir}/libtaos.*         || :
     ${csudo} rm -f ${lib64_link_dir}/libtaos.*       || :
 
+    ${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/*
+
     ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1
     ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
 

source/common/src/ttime.c

@@ -406,7 +406,31 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
     default: {
       return -1;
     }
-  }  
+  }
+}
+
+int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) {
+  int32_t charLen = varDataLen(inputData);
+  char *newColData;
+  if (type == TSDB_DATA_TYPE_BINARY) {
+    newColData = taosMemoryCalloc(1,  charLen + 1);
+    memcpy(newColData, varDataVal(inputData), charLen);
+    taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0);
+    taosMemoryFree(newColData);
+  } else if (type == TSDB_DATA_TYPE_NCHAR) {
+    newColData = taosMemoryCalloc(1,  charLen / TSDB_NCHAR_SIZE + 1);
+    int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inputData), charLen, newColData);
+    if (len < 0){
+      taosMemoryFree(newColData);
+      return TSDB_CODE_FAILED;
+    }
+    newColData[len] = 0;
+    taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0);
+    taosMemoryFree(newColData);
+  } else {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {

source/libs/function/inc/builtins.h

@@ -22,7 +22,7 @@ extern "C" {
 
 #include "functionMgt.h"
 
-#define FUNCTION_NAME_MAX_LENGTH 16
+#define FUNCTION_NAME_MAX_LENGTH 32
 
 #define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n)    (1 << n)
 

source/libs/function/src/builtins.c

@@ -403,6 +403,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .sprocessFunc = toISO8601Function,
     .finalizeFunc = NULL
   },
+  {
+    .name = "to_unixtimestamp",
+    .type = FUNCTION_TYPE_TO_UNIXTIMESTAMP,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .checkFunc    = checkAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = toUnixtimestampFunction,
+    .finalizeFunc = NULL
+  },
   {
     .name = "_rowts",
     .type = FUNCTION_TYPE_ROWTS,
@@ -622,6 +632,9 @@ int32_t checkAndGetResultType(SFunctionNode* pFunc) {
     case FUNCTION_TYPE_TO_ISO8601: {
       pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY};
     }
+    case FUNCTION_TYPE_TO_UNIXTIMESTAMP: {
+      pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
+    }
 
     case FUNCTION_TYPE_TBNAME: {
       // todo

source/libs/scalar/inc/sclInt.h

@@ -46,8 +46,9 @@ typedef struct SScalarCtx {
 int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
 SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
 
-#define GET_PARAM_TYPE(_c)   ((_c)->columnData->info.type)
-#define GET_PARAM_BYTES(_c)  ((_c)->columnData->info.bytes)
+#define GET_PARAM_TYPE(_c)      ((_c)->columnData->info.type)
+#define GET_PARAM_BYTES(_c)     ((_c)->columnData->info.bytes)
+#define GET_PARAM_PRECISON(_c)  ((_c)->columnData->info.precision)
 
 void sclFreeParam(SScalarParam *param);
 

source/libs/scalar/src/sclfunc.c

 #include "function.h"
 #include "scalar.h"
 #include "tdatablock.h"
+#include "ttime.h"
 #include "sclInt.h"
 #include "sclvector.h"
 
@@ -872,6 +873,36 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  int32_t type = GET_PARAM_TYPE(pInput);
+  int32_t timePrec = GET_PARAM_PRECISON(pInput);
+  if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (inputNum != 1) {
+    return TSDB_CODE_FAILED;
+  }
+
+  char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0];
+  for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
+    if (colDataIsNull_s(pInput[0].columnData, i)) {
+      colDataAppendNULL(pOutput->columnData, i);
+      continue;
+    }
+
+    int64_t timeVal = 0;
+    convertStringToTimestamp(type, input, timePrec, &timeVal);
+
+    colDataAppend(pOutput->columnData, i, (char *)&timeVal, false);
+    input += varDataTLen(input);
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   return doScalarFunctionUnique(pInput, inputNum, pOutput, atan);
 }

tests/script/tsim/tmq/consumerMain.sim

-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
-# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
-# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
-# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
-######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
-######## This test case include scene1 and scene3
-######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
-
-system sh/stop_dnodes.sh
-system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
-system sh/exec.sh -n dnode1 -s start
-
-$loop_cnt = 0
-check_dnode_ready:
-	$loop_cnt = $loop_cnt + 1
-	sleep 200
-	if $loop_cnt == 10 then
-	  print ====> dnode not ready!
-		return -1
-	endi
-sql show dnodes
-print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
-if $data00 != 1 then
-  return -1
-endi
-if $data04 != ready then
-  goto check_dnode_ready
-endi
-
-sql connect
-
-$loop_cnt = 0
-$vgroups  = 1
-$dbNamme  = d0
-loop_vgroups:
-print =============== create database $dbNamme vgroups $vgroups
-sql create database $dbNamme vgroups $vgroups
-sql show databases
-print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
-print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
-print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
-
-if $loop_cnt == 0 then 
-  if $rows != 2 then 
-    return -1
-  endi
-  if $data02 != 1 then  # vgroups
-    print vgroups: $data02
-    return -1
-  endi
-else
-  if $rows != 3 then 
-    return -1
-  endi
-  if $data00 == d1 then 
-    if $data02 != 4 then  # vgroups
-      print vgroups: $data02
-      return -1
-    endi
-  else  
-    if $data12 != 4 then  # vgroups
-      print vgroups: $data12
-      return -1
-    endi
-  endi
-endi
-
-sql use $dbNamme
-
-print =============== create super table
-sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
-
-sql show stables
-if $rows != 1 then 
-  return -1
-endi
-
-print =============== create child table
-$tbPrefix = ct
-$tbNum = 2
-
-$i = 0
-while $i < $tbNum
-  $tb = $tbPrefix . $i
-  sql create table $tb using stb tags( $i )
-  $i = $i + 1
-endw
-
-print =============== create normal table
-sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
-
-print =============== create multi topics. notes: now only support:
-print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb
-print =============== will support: * from stb
-
-sql create topic topic_stb_column as select ts, c1, c3 from stb
-#sql create topic topic_stb_all as select * from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-
-sql create topic topic_ctb_column as select ts, c1, c3 from ct0
-sql create topic topic_ctb_all as select * from ct0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
-
-sql create topic topic_ntb_column as select ts, c1, c3 from ntb
-sql create topic topic_ntb_all as select * from ntb
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
-
-sql show tables
-if $rows != 3 then 
-  return -1
-endi
-
-print =============== run_back insert data
-
-if $loop_cnt == 0 then 
-  run_back tsim/tmq/insertDataV1.sim
-else
-  run_back tsim/tmq/insertDataV4.sim
-endi
-
-sleep 1000
-
-#$rowNum = 1000
-#$tstart = 1640966400000  # 2022-01-01 00:00:00.000
-#
-#$i = 0
-#while $i < $tbNum
-#  $tb = $tbPrefix . $i
-#
-#  $x = 0
-#  while $x < $rowNum
-#    $c = $x / 10
-#    $c = $c * 10
-#    $c = $x - $c
-#
-#    $binary = ' . binary
-#    $binary = $binary . $c
-#    $binary = $binary . '
-#
-#    sql insert into $tb values ($tstart , $c , $x , $binary )
-#    sql insert into ntb values ($tstart , $c , $x , $binary )
-#    $tstart = $tstart + 1
-#    $x = $x + 1
-#  endw
-#
-#  $i = $i + 1
-#  $tstart = 1640966400000
-#endw
-
-#root@trd02 /home $ tmq_sim --help                                                                          
-# -c      Configuration directory, default is 
-# -d      The name of the database for cosumer, no default 
-# -t      The topic string for cosumer, no default 
-# -k      The key-value string for cosumer, no default 
-# -g      showMsgFlag, default is 0
-#        
-
-$consumeDelay = 5000
-$expectConsumeMsgCnt = 1000
-print expectConsumeMsgCnt: $expectConsumeMsgCnt, consumeDelay: $consumeDelay
-
-# supported key:
-# group.id:<xxx>
-# enable.auto.commit:<true | false>
-# auto.offset.reset:<earliest | latest | none>
-# td.connect.ip:<fqdn | ipaddress>
-# td.connect.user:root
-# td.connect.pass:taosdata
-# td.connect.port:6030
-# td.connect.db:db
-
-$expect_result = @{consume success: @
-$expect_result = $expect_result . $expectConsumeMsgCnt
-$expect_result = $expect_result . @, @ 
-$expect_result = $expect_result . 0}
-print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-#print cmd result----> $system_content
-##if $system_content != @{consume success: 10000, 0}@ then
-#if $system_content != $expect_result then
-#  return -1
-#endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-#if $system_content != @{consume success: 10000, 0}@ then
-if $system_content >= $expect_result then
-  return -1
-endi
-
-$expect_result = @{consume success: @
-$expect_result = $expect_result . $rowNum
-$expect_result = $expect_result . @, @ 
-$expect_result = $expect_result . 0}
-print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-$expect_result = @{consume success: @
-$expect_result = $expect_result . $totalMsgCnt
-$expect_result = $expect_result . @, @ 
-$expect_result = $expect_result . 0}
-print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-if $loop_cnt == 0 then
-  $loop_cnt = 1
-  $vgroups  = 4
-  $dbNamme  = d1
-  goto loop_vgroups
-endi
-
-
-#system sh/exec.sh -n dnode1 -s stop -x SIGINT