fix(query): opt filter perf.

source/libs/scalar/inc/sclInt.h

@@ -50,9 +50,6 @@ typedef struct SScalarCtx {
 #define SCL_IS_COMPARISON_OPERATOR(_opType) ((_opType) >= OP_TYPE_GREATER_THAN && (_opType) < OP_TYPE_IS_NOT_UNKNOWN)
 #define SCL_DOWNGRADE_DATETYPE(_type) \
   ((_type) == TSDB_DATA_TYPE_BIGINT || TSDB_DATA_TYPE_DOUBLE == (_type) || (_type) == TSDB_DATA_TYPE_UBIGINT)
-#define SCL_NO_NEED_CONVERT_COMPARISION(_ltype, _rtype, _optr) \
-  (IS_NUMERIC_TYPE(_ltype) && IS_NUMERIC_TYPE(_rtype) &&       \
-   ((_optr) >= OP_TYPE_GREATER_THAN && (_optr) <= OP_TYPE_NOT_EQUAL))
 
 #define sclFatal(...) qFatal(__VA_ARGS__)
 #define sclError(...) qError(__VA_ARGS__)

source/libs/scalar/src/filter.c

@@ -4082,7 +4082,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC
     SArray *pList = taosArrayInit(1, POINTER_BYTES);
     taosArrayPush(pList, &pSrc);
 
-    int32_t code = scalarCalculate(info->sclCtx.node, pList, &output);
+    code = scalarCalculate(info->sclCtx.node, pList, &output);
     taosArrayDestroy(pList);
 
     FLT_ERR_RET(code);

source/libs/scalar/src/sclvector.c

@@ -37,6 +37,11 @@
 
 #define IS_HELPER_NULL(col, i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i))
 
+bool noConvertBeforeCompare(int32_t leftType, int32_t rightType, int32_t optr) {
+  return IS_NUMERIC_TYPE(leftType) && IS_NUMERIC_TYPE(rightType) &&
+         (optr >= OP_TYPE_GREATER_THAN && optr <= OP_TYPE_NOT_EQUAL);
+}
+
 void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType) {
   switch (outType) {
     case TSDB_DATA_TYPE_BOOL: {
@@ -338,6 +343,7 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd
   colDataAppendInt8(pOut->columnData, rowIndex, (int8_t *)&v);
 }
 
+// todo remove this malloc
 static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
   int32_t len = 0;
   int32_t inputLen = varDataLen(buf);
@@ -399,6 +405,8 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
   }
 
   pCtx->pOut->numOfRows = pCtx->pIn->numOfRows;
+  char* tmp = NULL;
+
   for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) {
     if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) {
       colDataAppendNULL(pCtx->pOut->columnData, i);
@@ -421,12 +429,16 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
         continue;
       }
     }
+
     int32_t bufSize = pCtx->pIn->columnData->info.bytes;
-    char   *tmp = taosMemoryMalloc(varDataTLen(data));
-    if (!tmp) {
-      sclError("out of memory in vectorConvertFromVarData");
-      return TSDB_CODE_OUT_OF_MEMORY;
+    if (tmp == NULL) {
+      tmp = taosMemoryMalloc(bufSize);
+      if (tmp == NULL) {
+        sclError("out of memory in vectorConvertFromVarData");
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
     }
+
     if (vton) {
       memcpy(tmp, data, varDataTLen(data));
     } else {
@@ -434,6 +446,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
         memcpy(tmp, varDataVal(data), varDataLen(data));
         tmp[varDataLen(data)] = 0;
       } else if (TSDB_DATA_TYPE_NCHAR == convertType) {
+        // we need to convert it to native char string, and then perform the string to numeric data
         ASSERT(varDataLen(data) <= bufSize);
 
         int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
@@ -448,9 +461,11 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
     }
 
     (*func)(tmp, pCtx->pOut, i, overflow);
-    taosMemoryFreeClear(tmp);
   }
 
+  if (tmp != NULL) {
+    taosMemoryFreeClear(tmp);
+  }
   return TSDB_CODE_SUCCESS;
 }
 
@@ -925,25 +940,43 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara
     return TSDB_CODE_SUCCESS;
   }
 
+  int8_t  type = 0;
+  int32_t code = 0;
+
   SScalarParam *param1 = NULL, *paramOut1 = NULL;
   SScalarParam *param2 = NULL, *paramOut2 = NULL;
-  int32_t       code = 0;
 
-  if (leftType < rightType) {
+  // always convert least data
+  if (IS_VAR_DATA_TYPE(leftType) && IS_VAR_DATA_TYPE(rightType) && (pLeft->numOfRows != pRight->numOfRows) &&
+      leftType != TSDB_DATA_TYPE_JSON && rightType != TSDB_DATA_TYPE_JSON) {
     param1 = pLeft;
     param2 = pRight;
     paramOut1 = pLeftOut;
     paramOut2 = pRightOut;
+
+    if (pLeft->numOfRows > pRight->numOfRows) {
+      type = leftType;
+    } else {
+      type = rightType;
+    }
   } else {
-    param1 = pRight;
-    param2 = pLeft;
-    paramOut1 = pRightOut;
-    paramOut2 = pLeftOut;
-  }
+    // we only define half value in the convert-matrix, so make sure param1 always less equal than param2
+    if (leftType < rightType) {
+      param1 = pLeft;
+      param2 = pRight;
+      paramOut1 = pLeftOut;
+      paramOut2 = pRightOut;
+    } else {
+      param1 = pRight;
+      param2 = pLeft;
+      paramOut1 = pRightOut;
+      paramOut2 = pLeftOut;
+    }
 
-  int8_t type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2));
-  if (0 == type) {
-    return TSDB_CODE_SUCCESS;
+    type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2));
+    if (0 == type) {
+      return TSDB_CODE_SUCCESS;
+    }
   }
 
   if (type != GET_PARAM_TYPE(param1)) {
@@ -1683,7 +1716,7 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *
   SScalarParam *param1 = NULL;
   SScalarParam *param2 = NULL;
 
-  if (SCL_NO_NEED_CONVERT_COMPARISION(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
+  if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
     param1 = pLeft;
     param2 = pRight;
   } else {

source/util/src/tcompare.c

@@ -1071,7 +1071,7 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const
   return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
 }
 
-int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) {
+int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) {
   TdUcs4 c, c1;
   TdUcs4 matchOne = L'_';  // "_"
   TdUcs4 matchAll = L'%';  // "%"
@@ -1079,10 +1079,10 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size,
   int32_t i = 0;
   int32_t j = 0;
 
-  while ((c = patterStr[i++]) != 0) {
+  while ((c = pattern[i++]) != 0) {
     if (c == matchAll) { /* Match "%" */
 
-      while ((c = patterStr[i++]) == matchAll || c == matchOne) {
+      while ((c = pattern[i++]) == matchAll || c == matchOne) {
         if (c == matchOne && (j >= size || str[j++] == 0)) {
           return TSDB_PATTERN_NOWILDCARDMATCH;
         }
@@ -1100,7 +1100,7 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size,
           break;
         }
 
-        int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
+        int32_t ret = WCSPatternMatch(&pattern[i], ++str, size - n - 1, pInfo);
         if (ret != TSDB_PATTERN_NOMATCH) {
           return ret;
         }
@@ -1198,7 +1198,7 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight) {
 int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
 
-  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
+  ASSERT(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
 
   char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1);
   memcpy(pattern, varDataVal(pRight), varDataLen(pRight));