TD-6129<feature> add tag-> select logic

src/client/inc/tscUtil.h

@@ -28,6 +28,12 @@ extern "C" {
 #include "tscGlobalmerge.h"
 #include "tsched.h"
 #include "tsclient.h"
+#include "tglobal.h"
+
+#define JSON_TYPE_BINARY (strcasecmp(tsDefaultJSONStrType, "binary") == 0)
+#define JSON_TYPE_NCHAR (strcasecmp(tsDefaultJSONStrType, "nchar") == 0)
+#define SELECT_ALL_JSON_TAG 1
+#define SELECT_ELEMENT_JSON_TAG 2
 
 #define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
@@ -376,6 +382,8 @@ char* cloneCurrentDBName(SSqlObj* pSql);
 
 int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId);
 char* parseTagDatatoJson(void *p);
+void findTagValue(void* data, char* key, int32_t keyLen, char* out);
+
 #ifdef __cplusplus
 }
 #endif

src/client/src/tscUtil.c

@@ -729,11 +729,15 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
       if (isNull(p, TSDB_DATA_TYPE_NCHAR)) {
         memcpy(dst, p, varDataTLen(p));
       } else {
-        char* json = parseTagDatatoJson(p);
-        if(json){
-          memcpy(varDataVal(dst), json, strlen(json));
-          varDataSetLen(dst, strlen(json));
-          tfree(json);
+        if (*p == SELECT_ALL_JSON_TAG){
+          char* json = parseTagDatatoJson(p+1);
+          if(json) {
+            memcpy(varDataVal(dst), json, strlen(json));
+            varDataSetLen(dst, strlen(json));
+            tfree(json);
+          }
+        }else if (*p == SELECT_ELEMENT_JSON_TAG){
+          memcpy(dst, p+1, varDataTLen(p+1));
         }else{
           tscError("construct json error");
         }
@@ -5174,9 +5178,7 @@ char* cloneCurrentDBName(SSqlObj* pSql) {
   return p;
 }
 
-pExprInfo[j].base.param[0].nLen
-
-char* findTagValue(void* data, char* key, int32_t keyLen){
+void findTagValue(void* data, char* key, int32_t keyLen, char* out){
   int16_t nCols = kvRowNCols(data);
 
   bool found = false;
@@ -5186,14 +5188,19 @@ char* findTagValue(void* data, char* key, int32_t keyLen){
 
     if (k % 2 != 0) {  // json key
       char    tagJsonKey[TSDB_MAX_TAGS_LEN] = {0};
-      int32_t length = taosUcs4ToMbs(varDataVal(result), varDataLen(result), tagJsonKey);
-      if (length == 0) {
-        tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
-                 (char*)result);
-        continue;
-      }
-      if (strncmp(key, tagJsonKey, keyLen) != 0) {
-        continue;
+      if (JSON_TYPE_BINARY){
+        if (keyLen != varDataLen(result)) continue;
+        if (memcmp(varDataVal(result), key, keyLen) != 0) continue;
+      } else if(JSON_TYPE_NCHAR){
+        int32_t length = taosUcs4ToMbs(varDataVal(result), varDataLen(result), tagJsonKey);
+        if (length == 0) {
+          tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+                   (char*)result);
+          continue;
+        }
+        if (strncmp(key, tagJsonKey, keyLen) != 0) {
+          continue;
+        }
       }
       found = true;
     } else {  // json value
@@ -5206,12 +5213,18 @@ char* findTagValue(void* data, char* key, int32_t keyLen){
         if (length == 0) {
           tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
                    (char*)result);
+        }else{
+          varDataSetLen(out, length);
+          memcpy(varDataVal(out), tagJsonValue, length);
         }
       } else if (*(char*)result == cJSON_Number) {
         double jsonVd = *(double*)(POINTER_SHIFT(result, CHAR_BYTES));
+        sprintf(varDataVal(out), "%.9lf", jsonVd);
+        varDataSetLen(out, strlen(varDataVal(out)));
       } else {
         tscError("unsupportted json value");
       }
+      break;
     }
   }
 }
@@ -5237,18 +5250,28 @@ char* parseTagDatatoJson(void *p){
     }
     if (j%2 != 0) { // json key
       memset(tagJsonKey, 0, TSDB_MAX_TAGS_LEN);
-      int32_t length = taosUcs4ToMbs(varDataVal(val), varDataLen(val), tagJsonKey);
-      if (length == 0) {
-        tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val);
-        goto end;
+      if (JSON_TYPE_BINARY){
+        strncpy(tagJsonKey, varDataVal(val), varDataLen(val));
+      } else if(JSON_TYPE_NCHAR){
+        int32_t length = taosUcs4ToMbs(varDataVal(val), varDataLen(val), tagJsonKey);
+        if (length == 0) {
+          tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val);
+          goto end;
+        }
       }
     }else{  // json value
       char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0};
+      char* realData = POINTER_SHIFT(val, CHAR_BYTES);
       if(*(char*)val == cJSON_String){
-        int32_t length = taosUcs4ToMbs(varDataVal(POINTER_SHIFT(val,CHAR_BYTES)), varDataLen(POINTER_SHIFT(val,CHAR_BYTES)), tagJsonValue);
-        if (length == 0) {
-          tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val);
-          goto end;
+        if (JSON_TYPE_BINARY){
+          strncpy(tagJsonValue, varDataVal(realData), varDataLen(realData), varDataLen(realData));
+        } else if(JSON_TYPE_NCHAR) {
+          int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue);
+          if (length == 0) {
+            tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+                     (char*)val);
+            goto end;
+          }
         }
         cJSON* value = cJSON_CreateString(tagJsonValue);
         if (value == NULL)
@@ -5257,7 +5280,7 @@ char* parseTagDatatoJson(void *p){
         }
         cJSON_AddItemToObject(json, tagJsonKey, value);
       }else if(*(char*)val == cJSON_Number){
-        double jsonVd = *(double*)(POINTER_SHIFT(val,CHAR_BYTES));
+        double jsonVd = *(double*)(realData);
         cJSON* value = cJSON_CreateNumber(jsonVd);
         if (value == NULL)
         {
@@ -5299,28 +5322,39 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
       goto end;
     }
     char tagVal[TSDB_MAX_TAGS_LEN] = {0};
-    int32_t output = 0;
-    if (!taosMbsToUcs4(item->string, strlen(item->string), varDataVal(tagVal), TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE, &output)) {
-      tscError("json string error:%s|%s", strerror(errno), item->string);
-      retCode =  tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
-      goto end;
+    int32_t outLen = 0;
+    if (JSON_TYPE_BINARY){
+      strncpy(tagVal, item->string, strlen(item->string));
+      outLen = strlen(item->string);
+    }else if(JSON_TYPE_NCHAR){
+      if (!taosMbsToUcs4(item->string, strlen(item->string), varDataVal(tagVal), TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE, &outLen)) {
+        tscError("json string error:%s|%s", strerror(errno), item->string);
+        retCode =  tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
+        goto end;
+      }
     }
 
-    varDataSetLen(tagVal, output);
+    varDataSetLen(tagVal, outLen);
     tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagVal, false);   // add json key
 
     memset(tagVal, 0, TSDB_MAX_TAGS_LEN);
     if(item->type == cJSON_String){     // add json value  format: type|data
-      output = 0;
+      outLen = 0;
       *tagVal = item->type;     // type
       char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
-      if (!taosMbsToUcs4(item->valuestring, strlen(item->valuestring), varDataVal(tagData), TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE, &output)) {
-        tscError("json string error:%s|%s", strerror(errno), item->string);
-        retCode =  tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
-        goto end;
+      if (JSON_TYPE_BINARY){
+        strncpy(tagVal, item->valuestring, strlen(item->valuestring));
+        outLen = strlen(item->valuestring);
+      }else if(JSON_TYPE_NCHAR) {
+        if (!taosMbsToUcs4(item->valuestring, strlen(item->valuestring), varDataVal(tagData),
+                           TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE, &output)) {
+          tscError("json string error:%s|%s", strerror(errno), item->string);
+          retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
+          goto end;
+        }
       }
 
-      varDataSetLen(tagData, output);
+      varDataSetLen(tagData, outLen);
       tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagVal, true);
     }else if(item->type == cJSON_Number){
       *tagVal = item->type;    // type

src/common/CMakeLists.txt

@@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 ADD_LIBRARY(common ${SRC})

src/common/src/tglobal.c

@@ -283,7 +283,7 @@ char     Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESS
 int8_t tsDeadLockKillQuery = 0;
 
 // default JSON string type
-char tsDefaultJSONStrType[7] = "binary";
+char tsDefaultJSONStrType[7] = "nchar";
 
 int32_t (*monStartSystemFp)() = NULL;
 void (*monStopSystemFp)() = NULL;

src/common/src/tvariant.c

@@ -22,6 +22,7 @@
 #include "ttype.h"
 #include "tutil.h"
 #include "tvariant.h"
+#include "tscUtil.h"
 
 #define SET_EXT_INFO(converted, res, minv, maxv, exti) do {                                                       \
                                                         if (converted == NULL || exti == NULL || *converted == false) { break; }  \
@@ -105,6 +106,10 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) {
  * @param type
  */
 void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type) {
+  if(type == TSDB_DATA_TYPE_JSON){
+    if(JSON_TYPE_BINARY) type = TSDB_DATA_TYPE_BINARY;
+    else if(JSON_TYPE_NCHAR) type = TSDB_DATA_TYPE_NCHAR;
+  }
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT: {
@@ -167,8 +172,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
       
       break;
     }
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_JSON:{  // todo refactor, extract a method
+    case TSDB_DATA_TYPE_BINARY:{
       pVar->pz = calloc(len + 1, sizeof(char));
       memcpy(pVar->pz, pz, len);
       pVar->nLen = (int32_t)len;

src/query/src/qExecutor.c

@@ -32,6 +32,7 @@
 #include "tscLog.h"
 #include "cJSON.h"
 #include "tsdbMeta.h"
+#include "tscUtil.h"
 
 #define IS_MASTER_SCAN(runtime)        ((runtime)->scanFlag == MASTER_SCAN)
 #define IS_REVERSE_SCAN(runtime)       ((runtime)->scanFlag == REVERSE_SCAN)
@@ -7174,24 +7175,30 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) {
         type  = pExprInfo[j].base.resType;
         bytes = pExprInfo[j].base.resBytes;
 
+        char* tagJsonElementData = NULL;
+        dst  = pColInfo->pData + count * pExprInfo[j].base.resBytes;
         if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
           data = tsdbGetTableName(item->pTable);
         } else {
           data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes);
-          if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation
-            STSchema *pSchema = tsdbGetTableTagSchema((STable*) item->pTable);
-            STColumn *pCol = tdGetColOfID(pSchema, pExprInfo[j].base.colInfo.colId);
-            if (pCol == NULL) {
-              continue;  // No matched tag volumn
-            }
-            if(pCol->type == TSDB_DATA_TYPE_JSON){
-
+          if(type == TSDB_DATA_TYPE_JSON){
+            if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation
+              tagJsonElementData = calloc(bytes, 1);
+              findTagValue(data, pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, tagJsonElementData);
+              *dst = SELECT_ELEMENT_JSON_TAG;   // select tag->element
+              dst++;
+            }else{
+              *dst = SELECT_ALL_JSON_TAG;   // select tag
+              dst++;
             }
           }
         }
-
-        dst  = pColInfo->pData + count * pExprInfo[j].base.resBytes;
-        doSetTagValueToResultBuf(dst, data, type, bytes);
+        if(tagJsonElementData != NULL){
+          doSetTagValueToResultBuf(dst, tagJsonElementData, type, bytes);
+          tfree(tagJsonElementData);
+        }else{
+          doSetTagValueToResultBuf(dst, data, type, bytes);
+        }
       }
 
       count += 1;

src/query/src/qSqlParser.c

@@ -22,6 +22,7 @@
 #include "ttoken.h"
 #include "ttokendef.h"
 #include "tutil.h"
+#include "tscUtil.h"
 
 SSqlInfo qSqlParse(const char *pStr) {
   void *pParser = ParseAlloc(malloc);
@@ -714,7 +715,7 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
   pField->type = i;
   pField->bytes = tDataTypes[i].bytes;
 
-  if (i == TSDB_DATA_TYPE_NCHAR) {
+  if (i == TSDB_DATA_TYPE_NCHAR || (i == TSDB_DATA_TYPE_JSON && JSON_TYPE_NCHAR) {
     /*
      * for nchar, the TOKENTYPE is the number of character, so the length is the
      * number of bytes in UCS-4 format, which is 4 times larger than the number of characters
@@ -731,7 +732,7 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
       }
       pField->bytes = (int16_t)bytes;
     }
-  } else if (i == TSDB_DATA_TYPE_BINARY) {
+  } else if (i == TSDB_DATA_TYPE_BINARY || (i == TSDB_DATA_TYPE_JSON && JSON_TYPE_BINARY) {
     /* for binary, the TOKENTYPE is the length of binary */
     if (type->type == 0) {
       pField->bytes = 0;