refactor: modify json compare in where condition

source/client/src/clientImpl.c

@@ -619,7 +619,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
     int32_t bytes = pResultInfo->fields[i].bytes;
 
     if (IS_VAR_DATA_TYPE(type)) {
-      if (pCol->offset[pResultInfo->current] != -1) {
+      if (!colDataIsNull_var(pCol,pResultInfo->current)) {
         char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
 
         pResultInfo->length[i] = varDataLen(pStart);
@@ -725,6 +725,76 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
       pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
       pResultInfo->row[i] = pResultInfo->pCol[i].pData;
     }
+
+    if (type == TSDB_DATA_TYPE_JSON) {
+      char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
+      if (p == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+
+      pResultInfo->convertBuf[i] = p;
+      int32_t len = 0;
+      SResultColumn* pCol = &pResultInfo->pCol[i];
+      for (int32_t j = 0; j < numOfRows; ++j) {
+        if (pCol->offset[j] != -1) {
+          char* pStart = pCol->offset[j] + pCol->pData;
+
+          int32_t jsonInnerType = *pStart;
+          char *jsonInnerData = pStart + CHAR_BYTES;
+          char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
+          if(jsonInnerType == TSDB_DATA_TYPE_NULL){
+            sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          }else if(jsonInnerType == TSDB_DATA_TYPE_JSON){
+            int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
+
+            if (length <= 0) {
+              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData));
+              length = 0;
+            }
+            varDataSetLen(dst, length);
+          }else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) {   // value -> "value"
+            *(char*)varDataVal(dst) = '\"';
+            int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst) + CHAR_BYTES);
+            if (length <= 0) {
+              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData));
+              length = 0;
+            }
+            varDataSetLen(dst, length + CHAR_BYTES*2);
+            *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
+          }else if(jsonInnerType == TSDB_DATA_TYPE_DOUBLE){
+            double jsonVd = *(double*)(jsonInnerData);
+            sprintf(varDataVal(dst), "%.9lf", jsonVd);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          }else if(jsonInnerType == TSDB_DATA_TYPE_BIGINT){
+            int64_t jsonVd = *(int64_t*)(jsonInnerData);
+            sprintf(varDataVal(dst), "%" PRId64, jsonVd);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          }else if(jsonInnerType == TSDB_DATA_TYPE_BOOL){
+            sprintf(varDataVal(dst), "%s", (*((char *)jsonInnerData) == 1) ? "true" : "false");
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          }else {
+            ASSERT(0);
+          }
+
+          if(len + varDataTLen(dst) > colLength[i]){
+            p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));
+            if (p == NULL) {
+              return TSDB_CODE_OUT_OF_MEMORY;
+            }
+
+            pResultInfo->convertBuf[i] = p;
+          }
+          p = pResultInfo->convertBuf[i] + len;
+          memcpy(p, dst, varDataTLen(dst));
+          pCol->offset[j] = len;
+          len += varDataTLen(dst);
+        }
+      }
+
+      pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
+      pResultInfo->row[i] = pResultInfo->pCol[i].pData;
+    }
   }
 
   return TSDB_CODE_SUCCESS;

source/libs/scalar/inc/filterInt.h

@@ -308,7 +308,6 @@ struct SFilterInfo {
 #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri)))
 #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
 #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
-#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
 #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
 
 #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
@@ -321,7 +320,6 @@ struct SFilterInfo {
 #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
-#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
 #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
 #define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
 #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)

source/libs/scalar/src/filter.c

@@ -63,22 +63,18 @@ OptrStr gOptrStr[] = {
 
 bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(maxv, minr);
-  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
   return result >= 0;
 }
 bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(maxv, minr);
-  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
   return result > 0;
 }
 bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(minv, maxr);
-  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
   return result <= 0;
 }
 bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(minv, maxr);
-  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
   return result < 0;
 }
 bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
@@ -1783,9 +1779,9 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
       bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
 
       fi->data = taosMemoryCalloc(1, bytes);
-    } else if (type != TSDB_DATA_TYPE_JSON){
+    } else{
       if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) {  //TIME RANGE
-/*      
+/*
         fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes);
         for (int32_t a = 0; a < dType->bytes; ++a) {
           int64_t *v = taosArrayGet(var->arr, a);
@@ -1796,31 +1792,27 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
       } else {
         fi->data = taosMemoryCalloc(1, sizeof(int64_t));
       }
-    } else{     // type == TSDB_DATA_TYPE_JSON
-      // fi->data = null;  use fi->desc as data, because json value is variable, so use tVariant (fi->desc)
     }
 
-    if(type != TSDB_DATA_TYPE_JSON) {
-      if (dType->type == type) {
-        assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type);
+    if (dType->type == type) {
+      assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type);
+    } else {
+      SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
+      out.columnData->info.type = type;
+      if (IS_VAR_DATA_TYPE(type)) {
+        out.columnData->info.bytes = bytes;
       } else {
-        SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
-        out.columnData->info.type = type;
-        if (IS_VAR_DATA_TYPE(type)) {
-          out.columnData->info.bytes = bytes;
-        } else {
-          out.columnData->info.bytes = tDataTypes[type].bytes;
-        }
-
-        // todo refactor the convert
-        int32_t code = doConvertDataType(var, &out);
-        if (code != TSDB_CODE_SUCCESS) {
-          qError("convert value to type[%d] failed", type);
-          return TSDB_CODE_TSC_INVALID_OPERATION;
-        }
+        out.columnData->info.bytes = tDataTypes[type].bytes;
+      }
 
-        memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes);
+      // todo refactor the convert
+      int32_t code = doConvertDataType(var, &out);
+      if (code != TSDB_CODE_SUCCESS) {
+        qError("convert value to type[%d] failed", type);
+        return TSDB_CODE_TSC_INVALID_OPERATION;
       }
+
+      memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes);
     }
 
     // match/nmatch for nchar type need convert from ucs4 to mbs
@@ -2561,11 +2553,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
     info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
     
     if (unit->right.type == FLD_TYPE_VALUE) {
-      if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){   // json value is tVariant
-        info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit);
-      }else{
-        info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
-      }
+      info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
     } else {
       info->cunits[i].valData = NULL;
     }
@@ -2891,18 +2879,8 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
   for (int32_t i = 0; i < numOfRows; ++i) {
     uint32_t uidx = info->groups[0].unitIdxs[0];
     void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
-    if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
-      if (!colData){  // for json->'key' is null
-        (*p)[i] = 1;
-      }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){  // for json is null
-        colData = POINTER_SHIFT(colData, CHAR_BYTES);
-        (*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL);
-      }else{
-        (*p)[i] = 0;
-      }
-    }else{
-      (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
-    }
+    (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
+
     if ((*p)[i] == 0) {
       all = false;
     }    
@@ -2926,19 +2904,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
     uint32_t uidx = info->groups[0].unitIdxs[0];
     void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
 
-    if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
-      if (!colData) {   // for json->'key' is not null
-        (*p)[i] = 0;
-      }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){   // for json is not null
-        colData = POINTER_SHIFT(colData, CHAR_BYTES);
-        (*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL);
-      }else{    // for json->'key' is not null
-        (*p)[i] = 1;
-      }
-    }else {
-      (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
-    }
-
+    (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
     if ((*p)[i] == 0) {
       all = false;
     }