Merge branch 'feature/TD-6129' of github.com:taosdata/TDengine into feature/TD-6129

src/client/src/tscParseInsert.c

@@ -396,7 +396,7 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha
       } else if (pToken->type != TK_STRING){
         tscInvalidOperationMsg(msg, "invalid json data", pToken->z);
       } else{
-        *((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER;
+        *((int8_t *)payload) = TSDB_DATA_JSON_PLACEHOLDER;
       }
       break;
 

src/client/src/tscSQLParser.c

@@ -5319,7 +5319,8 @@ _ret:
 
 
 
-int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) {
+int32_t
+validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) {
   if (pExpr == NULL) {
     return TSDB_CODE_SUCCESS;
   }
@@ -6120,7 +6121,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         return invalidOperationMsg(pMsg, msg14);
       }
 
-      int8_t tagVal = TSDB_DATA_BINARY_PLACEHOLDER;
+      int8_t tagVal = TSDB_DATA_JSON_PLACEHOLDER;
       tdAddColToKVRow(&kvRowBuilder, pTagsSchema->colId, pTagsSchema->type, &tagVal, false);
 
       code = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, pMsg, pTagsSchema->colId);
@@ -7688,11 +7689,8 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
                 return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
               }
             }
-            if(pSchema->type == TSDB_DATA_TYPE_JSON){
-              *((int8_t *)tagVal) = TSDB_DATA_BINARY_PLACEHOLDER;
-            }else {
-              ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
-            }
+            ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
+
             // check again after the convert since it may be converted from binary to nchar.
             if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
               int16_t len = varDataTLen(tagVal);

src/client/src/tscUtil.c

@@ -5237,17 +5237,20 @@ char* parseTagDatatoJson(void *p){
   }
 
   int16_t nCols = kvRowNCols(p);
-  ASSERT(nCols%2 == 1);
+  ASSERT(nCols%2 == 0);
   char tagJsonKey[TSDB_MAX_JSON_KEY_LEN + 1] = {0};
   for (int j = 0; j < nCols; ++j) {
     SColIdx * pColIdx = kvRowColIdxAt(p, j);
     void* val = (kvRowColVal(p, pColIdx));
-    if (j == 0){        // json value is the first
+    if (j == 0){
       int8_t jsonVal = *(int8_t*)val;
-      ASSERT(jsonVal == TSDB_DATA_BINARY_PLACEHOLDER);
+      ASSERT(jsonVal == TSDB_DATA_JSON_PLACEHOLDER);
       continue;
     }
-    if (j%2 != 0) { // json key  encode by binary
+    if (j == 1 && *(uint8_t*)val == TSDB_DATA_JSON_NULL){
+      break;
+    }
+    if (j%2 == 0) { // json key  encode by binary
       ASSERT(varDataLen(val) <= TSDB_MAX_JSON_KEY_LEN);
       memset(tagJsonKey, 0, sizeof(tagJsonKey));
       memcpy(tagJsonKey, varDataVal(val), varDataLen(val));
@@ -5298,20 +5301,28 @@ end:
 }
 
 int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId){
+  if (strtrim(json) == 0 || strcasecmp(json, "null") == 0){
+    int8_t typeVal = TSDB_DATA_JSON_NULL;
+    tdAddColToKVRow(kvRowBuilder, startColId++, TSDB_DATA_TYPE_JSON, &typeVal, false);   // add json type
+  }else{
+    int8_t typeVal = TSDB_DATA_JSON_OBJECT;
+    tdAddColToKVRow(kvRowBuilder, startColId++, TSDB_DATA_TYPE_JSON, &typeVal, false);   // add json type
+  }
+
   cJSON *root = cJSON_Parse(json);
   if (root == NULL){
     tscError("json parse error");
     return tscSQLSyntaxErrMsg(errMsg, "json parse error", NULL);
   }
 
-  int retCode = 0;
   int size = cJSON_GetArraySize(root);
-  if(!cJSON_IsObject(root) || size == 0){
+  if(!cJSON_IsObject(root)){
     tscError("json error invalide value");
-    retCode = tscSQLSyntaxErrMsg(errMsg, "json error invalide value", NULL);
-    goto end;
+    return tscSQLSyntaxErrMsg(errMsg, "json error invalide value", NULL);
   }
 
+  int retCode = 0;
+  SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
   int jsonIndex = ++startColId;
   for(int i = 0; i < size; i++) {
     cJSON* item = cJSON_GetArrayItem(root, i);
@@ -5320,29 +5331,42 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
       retCode =  tscSQLSyntaxErrMsg(errMsg, "json inner error", NULL);
       goto end;
     }
-    if(strlen(item->string) > TSDB_MAX_JSON_KEY_LEN){
+
+    char *jsonKey = item->string;
+    if(strtrim(jsonKey) == 0) continue;
+    if (item->type == cJSON_NULL){
+      continue;
+    }
+
+    if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){
       tscError("json key too long error");
       retCode =  tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL);
       goto end;
     }
+    if(taosHashGet(keyHash, jsonKey, strlen(jsonKey)) != NULL){
+      continue;
+    }
 
     // json key encode by binary
     char tagKey[TSDB_MAX_JSON_KEY_LEN + VARSTR_HEADER_SIZE] = {0};
     int32_t outLen = 0;
-    strncpy(varDataVal(tagKey), item->string, strlen(item->string));
-    outLen = strlen(item->string);
+    strncpy(varDataVal(tagKey), jsonKey, strlen(jsonKey));
+    outLen = strlen(jsonKey);
+    taosHashPut(keyHash, jsonKey, strlen(jsonKey), 0, CHAR_BYTES);
 
     varDataSetLen(tagKey, outLen);
     tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagKey, false);   // add json key
 
     if(item->type == cJSON_String){     // add json value  format: type|data
+      char *jsonValue = item->valuestring;
+      strtrim(jsonValue);
       outLen = 0;
       char tagVal[TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE + CHAR_BYTES] = {0};
       *tagVal = jsonType2DbType(0, item->type);     // type
       char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
-      if (!taosMbsToUcs4(item->valuestring, strlen(item->valuestring), varDataVal(tagData),
+      if (!taosMbsToUcs4(jsonValue, strlen(jsonValue), varDataVal(tagData),
                          TSDB_MAX_TAGS_LEN, &outLen)) {
-        tscError("json string error:%s|%s", strerror(errno), item->string);
+        tscError("json string error:%s|%s", strerror(errno), jsonValue);
         retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
         goto end;
       }
@@ -5356,13 +5380,21 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
       if(*tagVal == TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble;
       else if(*tagVal == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint;
       tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_BIGINT, tagVal, true);
-    }else{
+    }else if(item->type == cJSON_True || item->type == cJSON_False){
+      char tagVal[CHAR_BYTES + CHAR_BYTES] = {0};
+      *tagVal = jsonType2DbType(item->valueint, item->type);    // type
+      char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
+      *tagData = item->valueint;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_BOOL, tagVal, true);
+    }
+    else{
       retCode =  tscSQLSyntaxErrMsg(errMsg, "invalidate json value", NULL);
       goto end;
     }
   }
 
 end:
+  taosHashCleanup(keyHash);
   cJSON_Delete(root);
   return retCode;
 }

src/common/src/ttypes.c

@@ -441,8 +441,8 @@ void setVardataNull(void* val, int32_t type) {
     varDataSetLen(val, sizeof(int32_t));
     *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL;
   } else if (type == TSDB_DATA_TYPE_JSON) {
-    varDataSetLen(val, sizeof(int32_t));
-    *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL;
+    varDataSetLen(val, sizeof(int8_t));
+    *(uint8_t*) varDataVal(val) = TSDB_DATA_JSON_NULL;
   } else {
     assert(0);
   }

src/common/src/tvariant.c

@@ -928,7 +928,7 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc
       if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
         //*(int8_t *)payload = TSDB_DATA_TINYINT_NULL;
       } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY){
-        *((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER;
+        *((int8_t *)payload) = TSDB_DATA_JSON_PLACEHOLDER;
       } else if (pVariant->nType == TSDB_DATA_TYPE_JSON){   // select * from stable, set tag type to json，from setTagValue/tag_project_function
         memcpy(payload, pVariant->pz, pVariant->nLen);
       }else {

src/inc/taosdef.h

@@ -71,7 +71,9 @@ extern const int32_t TYPE_BYTES[16];
 #define TSDB_DATA_DOUBLE_NULL           0x7FFFFF0000000000L     // an NAN
 #define TSDB_DATA_NCHAR_NULL            0xFFFFFFFF
 #define TSDB_DATA_BINARY_NULL           0xFF
-#define TSDB_DATA_BINARY_PLACEHOLDER    0x7F
+#define TSDB_DATA_JSON_PLACEHOLDER      0x7F
+#define TSDB_DATA_JSON_NULL             0xFF
+#define TSDB_DATA_JSON_OBJECT           0x01
 
 #define TSDB_DATA_UTINYINT_NULL         0xFF
 #define TSDB_DATA_USMALLINT_NULL        0xFFFF

src/inc/taoserror.h

@@ -288,7 +288,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_QRY_INCONSISTAN               TAOS_DEF_ERROR_CODE(0, 0x070C)  //"File inconsistency in replica")
 #define TSDB_CODE_QRY_INVALID_TIME_CONDITION    TAOS_DEF_ERROR_CODE(0, 0x070D)  //"invalid time condition")
 #define TSDB_CODE_QRY_SYS_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x070E)  //"System error")
-#define TSDB_CODE_QRY_JSON_KEY_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x070F)  //"json tag key not exist")
+//#define TSDB_CODE_QRY_JSON_KEY_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x070F)  //"json tag key not exist")
 #define TSDB_CODE_QRY_JSON_KEY_TYPE_ERROR       TAOS_DEF_ERROR_CODE(0, 0x0710)  //"json tag key type not match")
 #define TSDB_CODE_QRY_JSON_KEY_NOT_STR_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0711)  //"json tag key must be string in match/nmatch")
 

src/inc/ttype.h

@@ -159,8 +159,9 @@ static FORCE_INLINE bool isNull(const void *val, int32_t type) {
       return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL;
     case TSDB_DATA_TYPE_DOUBLE:
       return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
-    case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_JSON:
+      return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_JSON_NULL;
+    case TSDB_DATA_TYPE_NCHAR:
       return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
     case TSDB_DATA_TYPE_BINARY:
       return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;

src/query/src/qFilter.c

@@ -1167,13 +1167,10 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
     memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
 
     void* data = getJsonTagValue(info->pTable, schema->name, TSDB_MAX_JSON_KEY_MD5_LEN, &schema->colId);
-    if(data == NULL) return TSDB_CODE_QRY_JSON_KEY_NOT_EXIST;
-    schema->type = *(char*)data;  // if exist json tag-> operation get type so that can set data if (tree->_node.optr == TSDB_RELATION_IN_IN) the next and set value in filterInitValFieldData
-    assert(schema->type > TSDB_DATA_TYPE_NULL && schema->type < TSDB_DATA_TYPE_JSON);
-//    if((tree->_node.optr == TSDB_RELATION_MATCH || tree->_node.optr == TSDB_RELATION_NMATCH)
-//        && schema->type != TSDB_DATA_TYPE_BINARY)
-//      return TSDB_CODE_QRY_JSON_KEY_NOT_STR_ERROR;
-
+    if(data != NULL){
+      schema->type = *(char*)data;  // if exist json tag-> operation get type so that can set data if (tree->_node.optr == TSDB_RELATION_IN_IN) the next and set value in filterInitValFieldData
+      assert(schema->type > TSDB_DATA_TYPE_NULL && schema->type < TSDB_DATA_TYPE_JSON);
+    }
     pLeft = pLeft->_node.pLeft;   // -> operation use left as input
   }else if(tree->_node.optr == TSDB_RELATION_QUESTION){
     SSchema* schema = pLeft->pSchema;

src/tsdb/src/tsdbMeta.c

@@ -1100,14 +1100,14 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
   if(pSTable->tagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){
     ASSERT(pSTable->tagSchema->numOfCols == 1);
     int16_t nCols = kvRowNCols(pTable->tagVal);
-    ASSERT(nCols%2 == 1);
+    ASSERT(nCols%2 == 0);
     for (int j = 0; j < nCols; ++j) {
-      if (j != 0 && j%2 == 0) continue; // jump value
+      if (j != 0 && j%2 != 0) continue; // jump value
       SColIdx * pColIdx = kvRowColIdxAt(pTable->tagVal, j);
       void* val = (kvRowColVal(pTable->tagVal, pColIdx));
       if (j == 0){        // json value is the first
         int8_t jsonVal = *(int8_t*)val;
-        ASSERT(jsonVal == TSDB_DATA_BINARY_PLACEHOLDER);
+        ASSERT(jsonVal == TSDB_DATA_JSON_PLACEHOLDER);
         continue;
       }
 
@@ -1173,14 +1173,14 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
   if(pSTable->tagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){
     ASSERT(pSTable->tagSchema->numOfCols == 1);
     int16_t nCols = kvRowNCols(pTable->tagVal);
-    ASSERT(nCols%2 == 1);
+    ASSERT(nCols%2 == 0);
     for (int j = 0; j < nCols; ++j) {
-      if (j != 0 && j%2 == 0) continue; // jump value
+      if (j != 0 && j%2 != 0) continue; // jump value
       SColIdx * pColIdx = kvRowColIdxAt(pTable->tagVal, j);
       void* val = (kvRowColVal(pTable->tagVal, pColIdx));
       if (j == 0){        // json value is the first
         int8_t jsonVal = *(int8_t*)val;
-        ASSERT(jsonVal == TSDB_DATA_BINARY_PLACEHOLDER);
+        ASSERT(jsonVal == TSDB_DATA_JSON_PLACEHOLDER);
         continue;
       }
 

src/util/src/terror.c

@@ -294,7 +294,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER,        "Query buffer limit ha
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN,              "File inconsistance in replica")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION,   "One valid time range condition expected")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR,                "System error")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_KEY_NOT_EXIST,       "json tag key not exist")
+//TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_KEY_NOT_EXIST,       "json tag key not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_KEY_TYPE_ERROR,      "json tag key type not match")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_KEY_NOT_STR_ERROR,   "json tag key must be string in match/nmatch")
 

tests/pytest/stable/json_tag.py

@@ -161,6 +161,9 @@ class TDTestCase:
         tdSql.query("select * from db_json_tag_test.jsons1 where jtag->'location' match 'jin'")
         tdSql.checkRows(2)
 
+        tdSql.query("select * from db_json_tag_test.jsons1 where datastr match 'json and jtag->'location' match 'jin'")
+        tdSql.checkRows(2)
+
         tdSql.error("select * from db_json_tag_test.jsons1 where jtag->'num' match '5'")
 
     def stop(self):