Merge branch 'feature/TD-6129' of github.com:taosdata/TDengine into feature/TD-6129

src/client/inc/tscUtil.h

@@ -378,8 +378,8 @@ char* cloneCurrentDBName(SSqlObj* pSql);
 
 int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId);
 char* parseTagDatatoJson(void *p);
-void findTagValue(STable* data, char* key, int32_t keyLen, char** out, int* len);
-void parseTagValue2Dst(char* result, char* dst);
+void getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* out, int16_t bytes);
+void getJsonTagValueAll(void* data, void* dst, int16_t bytes);
 
 int8_t jsonType2DbType(double data, int jsonType);
 void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId);

src/client/src/tscPrepare.c

@@ -905,8 +905,6 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU
       }
 
       varDataSetLen(data + param->offset, output);
-    } else if (param->type == TSDB_DATA_TYPE_JSON) {   // todo json
-
     }
   }
 

src/client/src/tscUtil.c

@@ -684,7 +684,7 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
       }
     }
 
-  } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) {
+  } else if (convertNchar && (pInfo->field.type == TSDB_DATA_TYPE_NCHAR || pInfo->field.type == TSDB_DATA_TYPE_JSON)) {
     // convert unicode to native code in a temporary buffer extra one byte for terminated symbol
     char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
     if(buffer == NULL)
@@ -697,9 +697,11 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
     for (int32_t k = 0; k < pRes->numOfRows; ++k) {
       char* dst = pRes->buffer[i] + k * pInfo->field.bytes;
 
-      if (isNull(p, TSDB_DATA_TYPE_NCHAR)) {
+      if (isNull(p, TSDB_DATA_TYPE_NCHAR) && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) {
         memcpy(dst, p, varDataTLen(p));
-      } else if (varDataLen(p) > 0) {
+      } else if(isNull(p, TSDB_DATA_TYPE_JSON && pInfo->field.type == TSDB_DATA_TYPE_JSON)) {
+        memcpy(dst, p, varDataTLen(p));
+      }else if (varDataLen(p) > 0) {
         int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst));
         varDataSetLen(dst, length);
 
@@ -713,42 +715,6 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
       p += pInfo->field.bytes;
     }
 
-    memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
-  }else if (pInfo->field.type == TSDB_DATA_TYPE_JSON) {
-    // convert unicode to native code in a temporary buffer extra one byte for terminated symbol
-    char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
-    if(buffer == NULL)
-      return ;
-    pRes->buffer[i] = buffer;
-    // string terminated char for binary data
-    memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows);
-
-    char* p = pRes->urow[i];
-    for (int32_t k = 0; k < pRes->numOfRows; ++k) {
-      char* dst = pRes->buffer[i] + k * pInfo->field.bytes;
-      char* realData = p + CHAR_BYTES;
-      if (*p == SELECT_ALL_JSON_TAG){
-        char* json = parseTagDatatoJson(realData);
-        if(json) {
-          memcpy(varDataVal(dst), json, strlen(json));
-          varDataSetLen(dst, strlen(json));
-          assert(varDataTLen(dst) <= pInfo->field.bytes);
-          tfree(json);
-        }else{
-          setNull(dst, TSDB_DATA_TYPE_JSON, 0);
-        }
-      }else if (*p == SELECT_ELEMENT_JSON_TAG){
-        if (isNull(realData, TSDB_DATA_TYPE_JSON)) {
-          memcpy(dst, realData, varDataTLen(realData));
-        }else{
-          parseTagValue2Dst(realData, dst);
-        }
-      }else{
-        tscError("construct json error");
-      }
-      p += pInfo->field.bytes;
-    }
-
     memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
   }
 }
@@ -5182,53 +5148,53 @@ char* cloneCurrentDBName(SSqlObj* pSql) {
   return p;
 }
 
-void findTagValue(STable* data, char* key, int32_t keyLen, char** out, int* len){
-  void* result = getJsonTagValue(data, key, keyLen, NULL);
+void getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* dst, int16_t bytes){
+  char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
+  jsonKeyMd5(key, keyLen, keyMd5);
+
+  void* result = getJsonTagValue(data, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, NULL);
   if (result == NULL){    // json key no result
+    setNull(dst, TSDB_DATA_TYPE_JSON, 0);
     return;
   }
 
-  *out = result;
+  char out[TSDB_MAX_TAGS_LEN] = {0};
   char* realData = POINTER_SHIFT(result, CHAR_BYTES);
   if(*(char*)result == TSDB_DATA_TYPE_NCHAR) {
-    *len = varDataTLen(realData) + CHAR_BYTES;
+    assert(varDataTLen(result) <= TSDB_MAX_TAGS_LEN);
+    varDataCopy(out, result);
   }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE) {
-    *len = DOUBLE_BYTES + CHAR_BYTES;
+    double jsonVd = *(double*)(realData);
+    sprintf(varDataVal(out), "%.9lf", jsonVd);
+    varDataSetLen(out, strlen(varDataVal(out)));
   }else if (*(char*)result == TSDB_DATA_TYPE_BIGINT) {
-    *len = LONG_BYTES + CHAR_BYTES;
+    int64_t jsonVd = *(int64_t*)(realData);
+    sprintf(varDataVal(out), "%" PRId64, jsonVd);
+    varDataSetLen(out, strlen(varDataVal(out)));
   }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) {
-    *len = CHAR_BYTES + CHAR_BYTES;
+    sprintf(varDataVal(out), "%s", (*((char *)realData) == 1) ? "true" : "false");
+    varDataSetLen(out, strlen(varDataVal(out)));
   }else {
-    tscError("unsupportted json value");
-    return;
+    assert(0);
   }
+
+  int32_t length = 0;
+  taosMbsToUcs4(varDataVal(out), varDataLen(out), varDataVal(dst), bytes - VARSTR_HEADER_SIZE, &length);
+  varDataSetLen(dst, length);
 }
 
-void parseTagValue2Dst(char* result, char* dst){
-  char* realData = POINTER_SHIFT(result, CHAR_BYTES);
-  if(*(char*)result == TSDB_DATA_TYPE_NCHAR) {
-    char    tagJsonValue[TSDB_MAX_TAGS_LEN] = {0};
-    int32_t length = taosUcs4ToMbs(varDataVal(realData),
-                                   varDataLen(realData), tagJsonValue);
-    if (length < 0) {
-      tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
-               (char*)result);
+void getJsonTagValueAll(void* data, void* dst, int16_t bytes) {
+  char* json = parseTagDatatoJson(data);
+  if(json == NULL){
+    setNull(dst, TSDB_DATA_TYPE_JSON, 0);
     return;
   }
+
+  assert(strlen(json) <= bytes);
+  int32_t length = 0;
+  taosMbsToUcs4(json, strlen(json), varDataVal(dst), bytes - VARSTR_HEADER_SIZE, &length);
   varDataSetLen(dst, length);
-    memcpy(varDataVal(dst), tagJsonValue, length);
-  }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE) {
-    double jsonVd = *(double*)(realData);
-    sprintf(varDataVal(dst), "%.9lf", jsonVd);
-    varDataSetLen(dst, strlen(varDataVal(dst)));
-  }else if (*(char*)result == TSDB_DATA_TYPE_BIGINT) {
-    int64_t jsonVd = *(int64_t*)(realData);
-    sprintf(varDataVal(dst), "%" PRId64, jsonVd);
-    varDataSetLen(dst, strlen(varDataVal(dst)));
-  }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) {
-    sprintf(varDataVal(dst), "%s", (*((char *)realData) == 1) ? "true" : "false");
-    varDataSetLen(dst, strlen(varDataVal(dst)));
-  }
+  tfree(json);
 }
 
 char* parseTagDatatoJson(void *p){

src/common/src/ttypes.c

@@ -510,13 +510,9 @@ void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems) {
       break;
     case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_BINARY:
-      for (int32_t i = 0; i < numOfElems; ++i) {
-        setVardataNull(POINTER_SHIFT(val, i * bytes), type);
-      }
-      break;
     case TSDB_DATA_TYPE_JSON:
       for (int32_t i = 0; i < numOfElems; ++i) {
-        *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_JSON_NULL;
+        setVardataNull(POINTER_SHIFT(val, i * bytes), type);
       }
       break;
     default: {

src/inc/ttype.h

@@ -160,7 +160,7 @@ static FORCE_INLINE bool isNull(const void *val, int32_t type) {
     case TSDB_DATA_TYPE_DOUBLE:
       return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
     case TSDB_DATA_TYPE_JSON:
-      return *(uint8_t *)val == TSDB_DATA_JSON_NULL;
+      return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_JSON_NULL;
     case TSDB_DATA_TYPE_NCHAR:
       return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
     case TSDB_DATA_TYPE_BINARY:

src/query/inc/qUtil.h

@@ -17,9 +17,6 @@
 
 #include "tbuffer.h"
 
-#define SELECT_ALL_JSON_TAG 1
-#define SELECT_ELEMENT_JSON_TAG 2
-
 #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid)     \
   do {                                               \
     assert(sizeof(_uid) == sizeof(uint64_t));        \

src/query/src/qExecutor.c

@@ -3358,11 +3358,9 @@ static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag,
     tVariantCreateFromBinary(tag, varDataVal(val), len, type);
     //tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type);
   } else if(type == TSDB_DATA_TYPE_JSON){
-    assert(kvRowLen(val) < bytes);
-    tVariantCreateFromBinary(tag, val, kvRowLen(val), type);
-    memcpy(tag->pz + 1, tag->pz, kvRowLen(val) - 1);    // move back 1 byte for select type
-    *(tag->pz) = SELECT_ALL_JSON_TAG;
-    tag->nLen++;
+    char jsonVal[TSDB_MAX_TAGS_LEN] = {0};
+    getJsonTagValueAll(val, jsonVal, TSDB_MAX_TAGS_LEN);
+    tVariantCreateFromBinary(tag, jsonVal, varDataTLen(jsonVal), type);
   }
   else {
     tVariantCreateFromBinary(tag, val, bytes, type);
@@ -4322,7 +4320,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
         *compLen += compSizes[col];
         compSizes[col] = htonl(compSizes[col]);
       } else {
-        memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows);   // todo json
+        memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows);
         data += pColRes->info.bytes * pRes->info.rows;
       }
     }
@@ -4335,10 +4333,6 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
         *compLen += compSizes[col];
         compSizes[col] = htonl(compSizes[col]);
       } else {
-        if(pColRes->info.type == TSDB_DATA_TYPE_JSON){      // todo json
-          //pColRes->info.bytes =
-
-        }
         memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows);
         data += pColRes->info.bytes * numOfRows;
       }
@@ -7146,7 +7140,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) {
       if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
         data = tsdbGetTableName(item->pTable);
       } else {
-        data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes);  //todo json
+        data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes);
       }
 
       doSetTagValueToResultBuf(output, data, type, bytes);
@@ -7189,17 +7183,9 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) {
           data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes);
           if(type == TSDB_DATA_TYPE_JSON){
             if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation
-              char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
-              jsonKeyMd5(pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, keyMd5);
-              char* strStr = NULL;
-              int len = 0;
-              findTagValue(item->pTable, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, &strStr, &len);
-              *dst++ = SELECT_ELEMENT_JSON_TAG;   // select tag->element
-              doSetTagValueToResultBuf(dst, strStr, type, len);
+              getJsonTagValueElment(item->pTable, pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, dst, bytes);
             }else{
-              *dst++ = SELECT_ALL_JSON_TAG;   // select tag
-              assert(kvRowLen(data) < bytes);
-              doSetTagValueToResultBuf(dst, data, type, bytes - CHAR_BYTES);
+              getJsonTagValueAll(data, dst, bytes);
             }
             continue;
           }

src/util/src/tutil.c

@@ -95,7 +95,7 @@ size_t strtrim(char *z) {
   int32_t j = 0;
 
   int32_t delta = 0;
-  while (z[j] == ' ') {
+  while (isspace(z[j])) {
     ++j;
   }
 
@@ -108,9 +108,9 @@ size_t strtrim(char *z) {
 
   int32_t stop = 0;
   while (z[j] != 0) {
-    if (z[j] == ' ' && stop == 0) {
+    if (isspace(z[j]) && stop == 0) {
       stop = j;
-    } else if (z[j] != ' ' && stop != 0) {
+    } else if (!isspace(z[j]) && stop != 0) {
       stop = 0;
     }
 

tests/pytest/stable/json_tag.py

@@ -173,7 +173,7 @@ class TDTestCase:
 
         # test json string parse
         tdSql.error("CREATE TABLE if not exists db_json_tag_test.jsons1_5 using db_json_tag_test.jsons1 tags('efwewf')")
-        tdSql.error("CREATE TABLE if not exists db_json_tag_test.jsons1_5 using db_json_tag_test.jsons1 tags('\t')")
+        tdSql.execute("CREATE TABLE if not exists db_json_tag_test.jsons1_5 using db_json_tag_test.jsons1 tags('\t')")
         tdSql.execute("CREATE TABLE if not exists db_json_tag_test.jsons1_6 using db_json_tag_test.jsons1 tags('')")
         tdSql.query("select jtag from db_json_tag_test.jsons1_6")
         tdSql.checkData(0, 0, "")
@@ -204,7 +204,7 @@ class TDTestCase:
         tdSql.checkRows(1)
 
         tdSql.query("select jtag from db_json_tag_test.jsons1 where jtag is null")
-        tdSql.checkRows(4)
+        tdSql.checkRows(5)
 
         tdSql.query("select jtag from db_json_tag_test.jsons1 where jtag is not null")
         tdSql.checkRows(5)
@@ -213,7 +213,7 @@ class TDTestCase:
         tdSql.checkRows(3)
 
         tdSql.query("select tbname,jtag from db_json_tag_test.jsons1 where jtag->'location' is null")
-        tdSql.checkRows(6)
+        tdSql.checkRows(7)
 
         tdSql.query("select * from db_json_tag_test.jsons1 where jtag->'num' is not null")
         tdSql.checkRows(2)