TD-6129<feature> move select json->'key' decode logic to client

src/client/inc/tscUtil.h

@@ -378,7 +378,8 @@ char* cloneCurrentDBName(SSqlObj* pSql);
 
 int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId);
 char* parseTagDatatoJson(void *p);
-void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len);
+void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int* len);
+void parseTagValue2Dst(char* result, char* dst);
 
 int8_t jsonType2DbType(double data, int jsonType);
 void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId);

src/client/src/tscLocal.c

@@ -91,10 +91,13 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     if (pSchema[i].type == TSDB_DATA_TYPE_BINARY){
       bytes -= VARSTR_HEADER_SIZE;
     }
-    else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
+    else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
       bytes -= VARSTR_HEADER_SIZE;
       bytes = bytes / TSDB_NCHAR_SIZE;
     }
+    else if(pSchema[i].type == TSDB_DATA_TYPE_JSON) {
+      bytes = bytes / TSDB_NCHAR_SIZE;
+    }
 
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2);
     *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes;

src/client/src/tscParseInsert.c

@@ -1089,6 +1089,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
 //        tscDestroyBoundColumnInfo(&spd);
 //        return tscSQLSyntaxErrMsg(pInsertParam->msg, "json type error, should be string", NULL);
 //      }
+      if(sToken.n > TSDB_MAX_TAGS_LEN/TSDB_NCHAR_SIZE){
+        tdDestroyKVRowBuilder(&kvRowBuilder);
+        tscDestroyBoundColumnInfo(&spd);
+        return tscSQLSyntaxErrMsg(pInsertParam->msg, "json tag too long", NULL);
+      }
       code = parseJsontoTagData(sToken.z, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId);
       if (code != TSDB_CODE_SUCCESS) {
         tdDestroyKVRowBuilder(&kvRowBuilder);

src/client/src/tscSQLParser.c

@@ -6159,6 +6159,11 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         tdDestroyKVRowBuilder(&kvRowBuilder);
         return invalidOperationMsg(pMsg, msg25);
       }
+      if(pItem->pVar.nType > TSDB_MAX_TAGS_LEN/TSDB_NCHAR_SIZE){
+        tscError("json tag too long");
+        tdDestroyKVRowBuilder(&kvRowBuilder);
+        return invalidOperationMsg(pMsg, msg14);
+      }
 
       int8_t tagVal = TSDB_DATA_BINARY_PLACEHOLDER;
       tdAddColToKVRow(&kvRowBuilder, pTagsSchema->colId, pTagsSchema->type, &tagVal, false);
@@ -7763,11 +7768,8 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
             return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
           }
         }
-        if(pSchema->type == TSDB_DATA_TYPE_JSON){
-          *((int8_t *)tagVal) = TSDB_DATA_BINARY_PLACEHOLDER;
-        }else {
-          ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
-        }
+        ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
+
         // check again after the convert since it may be converted from binary to nchar.
         if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
           int16_t len = varDataTLen(tagVal);
@@ -7798,6 +7800,11 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
         tdDestroyKVRowBuilder(&kvRowBuilder);
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
       }
+      if(pItem->pVar.nLen > TSDB_MAX_TAGS_LEN/TSDB_NCHAR_SIZE){
+        tscError("json tag too long");
+        tdDestroyKVRowBuilder(&kvRowBuilder);
+        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
+      }
       ret = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, tscGetErrorMsgPayload(pCmd), pTagSchema[0].colId);
       if (ret != TSDB_CODE_SUCCESS) {
         tdDestroyKVRowBuilder(&kvRowBuilder);

src/client/src/tscUtil.c

@@ -736,7 +736,7 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
           tfree(json);
         }
       }else if (*p == SELECT_ELEMENT_JSON_TAG){
-        memcpy(dst, realData, varDataTLen(realData));
+        parseTagValue2Dst(realData, dst);
       }else{
         tscError("construct json error");
       }
@@ -5176,13 +5176,32 @@ char* cloneCurrentDBName(SSqlObj* pSql) {
   return p;
 }
 
-void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len){
+void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int* len){
   void* result = getJsonTagValue(data, key, keyLen, NULL);
   if (result == NULL){    // json key no result
     return;
   }
+
+  out = result;
   char* realData = POINTER_SHIFT(result, CHAR_BYTES);
+  if(*(char*)result == TSDB_DATA_TYPE_NCHAR) {
+    *len = varDataTLen(realData) + CHAR_BYTES;
+  }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE) {
+    *len = DOUBLE_BYTES + CHAR_BYTES;
+  }else if (*(char*)result == TSDB_DATA_TYPE_BIGINT) {
+    *len = LONG_BYTES + CHAR_BYTES;
+  }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) {
+    *len = CHAR_BYTES + CHAR_BYTES;
+  }else if (*(char*)result == TSDB_DATA_TYPE_NULL) {
+    *len = CHAR_BYTES + CHAR_BYTES;
+  }else {
+    tscError("unsupportted json value");
+    return;
+  }
+}
 
+void parseTagValue2Dst(char* result, char* dst){
+  char* realData = POINTER_SHIFT(result, CHAR_BYTES);
   if(*(char*)result == TSDB_DATA_TYPE_NCHAR) {
     char    tagJsonValue[TSDB_MAX_TAGS_LEN] = {0};
     int32_t length = taosUcs4ToMbs(varDataVal(realData),
@@ -5192,22 +5211,20 @@ void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t le
                (char*)result);
       return;
     }
-    assert(length <= len);
-    varDataSetLen(out, length);
-    memcpy(varDataVal(out), tagJsonValue, length);
+    varDataSetLen(dst, length);
+    memcpy(varDataVal(dst), tagJsonValue, length);
   }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE) {
     double jsonVd = *(double*)(realData);
-    sprintf(varDataVal(out), "%.9lf", jsonVd);
-    assert(strlen(varDataVal(out)) <= len);
-    varDataSetLen(out, strlen(varDataVal(out)));
+    sprintf(varDataVal(dst), "%.9lf", jsonVd);
+    varDataSetLen(dst, strlen(varDataVal(dst)));
   }else if (*(char*)result == TSDB_DATA_TYPE_BIGINT) {
     int64_t jsonVd = *(int64_t*)(realData);
-    sprintf(varDataVal(out), "%" PRId64, jsonVd);
-    assert(strlen(varDataVal(out)) <= len);
-    varDataSetLen(out, strlen(varDataVal(out)));
-  }
-  else {
-    tscError("unsupportted json value");
+    sprintf(varDataVal(dst), "%" PRId64, jsonVd);
+    varDataSetLen(dst, strlen(varDataVal(dst)));
+  }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) {
+
+  }else if (*(char*)result == TSDB_DATA_TYPE_NULL) {
+
   }
 }
 
@@ -5303,9 +5320,9 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
       retCode =  tscSQLSyntaxErrMsg(errMsg, "json inner error", NULL);
       goto end;
     }
-    if(strlen(item->string) >= TSDB_MAX_JSON_KEY_LEN){
+    if(strlen(item->string) > TSDB_MAX_JSON_KEY_LEN){
       tscError("json key too long error");
-      retCode =  tscSQLSyntaxErrMsg(errMsg, "json key too long, exceed 256", NULL);
+      retCode =  tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL);
       goto end;
     }
 

src/common/src/ttypes.c

@@ -18,7 +18,7 @@
 #include "ttokendef.h"
 #include "tscompression.h"
 
-const int32_t TYPE_BYTES[17] = {
+const int32_t TYPE_BYTES[16] = {
     -1,                      // TSDB_DATA_TYPE_NULL
     sizeof(int8_t),          // TSDB_DATA_TYPE_BOOL
     sizeof(int8_t),          // TSDB_DATA_TYPE_TINYINT
@@ -384,7 +384,7 @@ tDataTypeDescriptor tDataTypes[16] = {
   {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES,  "SMALLINT UNSIGNED",  0,          UINT16_MAX,     tsCompressSmallint,  tsDecompressSmallint,  getStatics_u16},
   {TSDB_DATA_TYPE_UINT,      12, INT_BYTES,    "INT UNSIGNED",       0,          UINT32_MAX,     tsCompressInt,       tsDecompressInt,       getStatics_u32},
   {TSDB_DATA_TYPE_UBIGINT,   15, LONG_BYTES,   "BIGINT UNSIGNED",    0,          UINT64_MAX,     tsCompressBigint,    tsDecompressBigint,    getStatics_u64},
-  {TSDB_DATA_TYPE_JSON,4,  0,     "JSON",               0,          0,              tsCompressString,    tsDecompressString,    getStatics_nchr},
+  {TSDB_DATA_TYPE_JSON,4,  JSON_BYTES,     "JSON",               0,          0,              tsCompressString,    tsDecompressString,    getStatics_nchr},
 };
 
 char tTokenTypeSwitcher[13] = {
@@ -440,6 +440,9 @@ void setVardataNull(void* val, int32_t type) {
   } else if (type == TSDB_DATA_TYPE_NCHAR) {
     varDataSetLen(val, sizeof(int32_t));
     *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL;
+  } else if (type == TSDB_DATA_TYPE_JSON) {
+    varDataSetLen(val, sizeof(int32_t));
+    *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL;
   } else {
     assert(0);
   }
@@ -507,6 +510,7 @@ void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems) {
       break;
     case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_JSON:
       for (int32_t i = 0; i < numOfElems; ++i) {
         setVardataNull(POINTER_SHIFT(val, i * bytes), type);
       }

src/common/src/tvariant.c

@@ -158,8 +158,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
       pVar->dKey = GET_FLOAT_VAL(pz);
       break;
     }
-    case TSDB_DATA_TYPE_NCHAR:
-    case TSDB_DATA_TYPE_JSON:{ // here we get the nchar length from raw binary bits length
+    case TSDB_DATA_TYPE_NCHAR:{ // here we get the nchar length from raw binary bits length
       size_t lenInwchar = len / TSDB_NCHAR_SIZE;
 
       pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE);
@@ -168,6 +167,12 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
       
       break;
     }
+    case TSDB_DATA_TYPE_JSON:{
+      pVar->pz = calloc(len + 2, sizeof(char));
+      memcpy(pVar->pz, pz, len);
+      pVar->nLen = (int32_t)len;
+      break;
+    }
     case TSDB_DATA_TYPE_BINARY:{
       pVar->pz = calloc(len + 1, sizeof(char));
       memcpy(pVar->pz, pz, len);
@@ -923,7 +928,7 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc
       if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
         //*(int8_t *)payload = TSDB_DATA_TINYINT_NULL;
       } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY){
-        //*((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER;
+        *((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER;
       } else if (pVariant->nType == TSDB_DATA_TYPE_JSON){   // select * from stable, set tag type to json，from setTagValue/tag_project_function
         memcpy(payload, pVariant->pz, pVariant->nLen);
       }else {

src/inc/taosdef.h

@@ -39,7 +39,7 @@ extern "C" {
 #define TSKEY_INITIAL_VAL    INT64_MIN
 
 // Bytes for each type.
-extern const int32_t TYPE_BYTES[17];
+extern const int32_t TYPE_BYTES[16];
 
 // TODO: replace and remove code below
 #define CHAR_BYTES    sizeof(char)
@@ -50,6 +50,7 @@ extern const int32_t TYPE_BYTES[17];
 #define DOUBLE_BYTES  sizeof(double)
 #define POINTER_BYTES sizeof(void *)  // 8 by default  assert(sizeof(ptrdiff_t) == sizseof(void*)
 
+#define JSON_BYTES (TSDB_MAX_TAGS_LEN)
 #define TSDB_KEYSIZE            sizeof(TSKEY)
 
 #if LINUX

src/inc/ttype.h

@@ -160,6 +160,7 @@ static FORCE_INLINE bool isNull(const void *val, int32_t type) {
     case TSDB_DATA_TYPE_DOUBLE:
       return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
     case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:
       return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
     case TSDB_DATA_TYPE_BINARY:
       return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;

src/query/src/qExecutor.c

@@ -3359,7 +3359,7 @@ static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag,
     //tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type);
   } else if(type == TSDB_DATA_TYPE_JSON){
     assert(kvRowLen(val) < bytes);
-    tVariantCreateFromBinary(tag, val, bytes, type);
+    tVariantCreateFromBinary(tag, val, kvRowLen(val), type);
     memcpy(tag->pz + 1, tag->pz, bytes - 1);    // move back 1 byte for select type
     *(tag->pz) = SELECT_ALL_JSON_TAG;
     tag->nLen++;
@@ -7189,18 +7189,15 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) {
           data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes);
           if(type == TSDB_DATA_TYPE_JSON){
             if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation
-              char* tagJsonElementData = calloc(bytes, 1);
               char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
               jsonKeyMd5(pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, keyMd5);
-              findTagValue(item->pTable, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, tagJsonElementData, bytes);
-              *dst = SELECT_ELEMENT_JSON_TAG;   // select tag->element
-              dst++;
-              assert(varDataTLen(tagJsonElementData) < bytes);
-              doSetTagValueToResultBuf(dst, tagJsonElementData, type, bytes - CHAR_BYTES);
-              tfree(tagJsonElementData);
+              char* strStr = NULL;
+              int len = 0;
+              findTagValue(item->pTable, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, strStr, &len);
+              *dst++ = SELECT_ELEMENT_JSON_TAG;   // select tag->element
+              doSetTagValueToResultBuf(dst, strStr, type, len);
             }else{
-              *dst = SELECT_ALL_JSON_TAG;   // select tag
-              dst++;
+              *dst++ = SELECT_ALL_JSON_TAG;   // select tag
               assert(kvRowLen(data) < bytes);
               doSetTagValueToResultBuf(dst, data, type, bytes - CHAR_BYTES);
             }

src/query/src/qSqlParser.c

@@ -716,7 +716,7 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
   pField->type = i;
   pField->bytes = tDataTypes[i].bytes;
 
-  if (i == TSDB_DATA_TYPE_NCHAR || i == TSDB_DATA_TYPE_JSON) {
+  if (i == TSDB_DATA_TYPE_NCHAR) {
     /*
      * for nchar, the TOKENTYPE is the number of character, so the length is the
      * number of bytes in UCS-4 format, which is 4 times larger than the number of characters