Merge pull request #12873 from taosdata/feature/TD-13041

feat:add json function

include/util/taoserror.h

@@ -639,6 +639,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_NOT_ALLOWED_FUNC          TAOS_DEF_ERROR_CODE(0, 0x264F)
 #define TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY     TAOS_DEF_ERROR_CODE(0, 0x2650)
 #define TSDB_CODE_PAR_INVALID_DROP_COL          TAOS_DEF_ERROR_CODE(0, 0x2651)
+#define TSDB_CODE_PAR_INVALID_COL_JSON          TAOS_DEF_ERROR_CODE(0, 0x2652)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tdef.h

@@ -234,6 +234,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
 #define TSDB_MAX_JSON_TAG_LEN 16384
+#define TSDB_MAX_JSON_KEY_LEN 256
 
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32

source/client/src/clientImpl.c

@@ -866,8 +866,7 @@ static char* parseTagDatatoJson(void* p) {
     if (j == 0) {
       if (*val == TSDB_DATA_TYPE_NULL) {
         string = taosMemoryCalloc(1, 8);
-        sprintf(varDataVal(string), "%s", TSDB_DATA_NULL_STR_L);
-        varDataSetLen(string, strlen(varDataVal(string)));
+        sprintf(string, "%s", TSDB_DATA_NULL_STR_L);
         goto end;
       }
       continue;
@@ -1003,7 +1002,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
               length = 0;
             }
             varDataSetLen(dst, length + CHAR_BYTES * 2);
-            *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
+            *(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"';
           } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
             double jsonVd = *(double*)(jsonInnerData);
             sprintf(varDataVal(dst), "%.9lf", jsonVd);

source/dnode/vnode/src/meta/metaTable.c

@@ -607,31 +607,39 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   if (iCol == 0) {
     // TODO : need to update tag index
   }
-
   ctbEntry.version = version;
-  SKVRowBuilder kvrb = {0};
-  const SKVRow  pOldTag = (const SKVRow)ctbEntry.ctbEntry.pTags;
-  SKVRow        pNewTag = NULL;
-
-  tdInitKVRowBuilder(&kvrb);
-  for (int32_t i = 0; i < pTagSchema->nCols; i++) {
-    SSchema *pCol = &pTagSchema->pSchema[i];
-    if (iCol == i) {
-      tdAddColToKVRow(&kvrb, pCol->colId, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
-    } else {
-      void *p = tdGetKVRowValOfCol(pOldTag, pCol->colId);
-      if (p) {
-        if (IS_VAR_DATA_TYPE(pCol->type)) {
-          tdAddColToKVRow(&kvrb, pCol->colId, p, varDataTLen(p));
-        } else {
-          tdAddColToKVRow(&kvrb, pCol->colId, p, pCol->bytes);
+  if(pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON){
+    ctbEntry.ctbEntry.pTags = taosMemoryMalloc(pAlterTbReq->nTagVal);
+    if(ctbEntry.ctbEntry.pTags == NULL){
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+    memcpy((void*)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
+  }else{
+    SKVRowBuilder kvrb = {0};
+    const SKVRow  pOldTag = (const SKVRow)ctbEntry.ctbEntry.pTags;
+    SKVRow        pNewTag = NULL;
+
+    tdInitKVRowBuilder(&kvrb);
+    for (int32_t i = 0; i < pTagSchema->nCols; i++) {
+      SSchema *pCol = &pTagSchema->pSchema[i];
+      if (iCol == i) {
+        tdAddColToKVRow(&kvrb, pCol->colId, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
+      } else {
+        void *p = tdGetKVRowValOfCol(pOldTag, pCol->colId);
+        if (p) {
+          if (IS_VAR_DATA_TYPE(pCol->type)) {
+            tdAddColToKVRow(&kvrb, pCol->colId, p, varDataTLen(p));
+          } else {
+            tdAddColToKVRow(&kvrb, pCol->colId, p, pCol->bytes);
+          }
         }
       }
     }
-  }
 
-  ctbEntry.ctbEntry.pTags = tdGetKVRowFromBuilder(&kvrb);
-  tdDestroyKVRowBuilder(&kvrb);
+    ctbEntry.ctbEntry.pTags = tdGetKVRowFromBuilder(&kvrb);
+    tdDestroyKVRowBuilder(&kvrb);
+  }
 
   // save to table.db
   metaSaveToTbDb(pMeta, &ctbEntry);
@@ -641,6 +649,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
 
   tDecoderClear(&dc1);
   tDecoderClear(&dc2);
+  if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void*)ctbEntry.ctbEntry.pTags);
   if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
   if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
   tdbTbcClose(pTbDbc);

source/libs/command/src/command.c

@@ -21,6 +21,7 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
     case TSDB_DATA_TYPE_BINARY:
       return (pSchema->bytes - VARSTR_HEADER_SIZE);
     case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:
       return (pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
     default:
       return pSchema->bytes;

source/libs/nodes/src/nodesUtilFuncs.c

@@ -1119,6 +1119,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
 bool nodesIsJsonOp(const SOperatorNode* pOp) {
   switch (pOp->opType) {
     case OP_TYPE_JSON_GET_VALUE:
+    case OP_TYPE_JSON_CONTAINS:
       return true;
     default:
       break;

source/libs/parser/src/parTranslater.c

@@ -677,7 +677,6 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
         varDataSetLen(pVal->datum.p, len);
         break;
       }
-      case TSDB_DATA_TYPE_JSON:
       case TSDB_DATA_TYPE_DECIMAL:
       case TSDB_DATA_TYPE_BLOB:
         return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
@@ -738,8 +737,7 @@ static EDealRes translateUnaryOperator(STranslateContext* pCxt, SOperatorNode* p
 static EDealRes translateArithmeticOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
   SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
   SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
-  if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
-      TSDB_DATA_TYPE_BLOB == rdt.type) {
+  if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
   }
   if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) ||
@@ -764,14 +762,14 @@ static EDealRes translateArithmeticOperator(STranslateContext* pCxt, SOperatorNo
 static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
   SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
   SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
-  if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
+  if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
   }
   if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) {
     ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType;
   }
   if (nodesIsRegularOp(pOp)) {
-    if (!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) {
+    if (!IS_VAR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName);
     }
     if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || !IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) {
@@ -2485,6 +2483,9 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SN
         code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN);
       }
     }
+    if (TSDB_CODE_SUCCESS == code && pCol->dataType.type == TSDB_DATA_TYPE_JSON) {
+      code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COL_JSON);
+    }
     int32_t len = strlen(pCol->colName);
     if (TSDB_CODE_SUCCESS == code && NULL != taosHashGet(pHash, pCol->colName, len)) {
       code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN);
@@ -2492,7 +2493,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SN
     if (TSDB_CODE_SUCCESS == code) {
       if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) ||
           (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN)) {
-        code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
+        code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
       }
     }
     if (TSDB_CODE_SUCCESS == code) {
@@ -4451,11 +4452,38 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
   }
 
   pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type);
-  pReq->nTagVal = pStmt->pVal->node.resType.bytes;
-  if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) {
-    pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE;
+  if(pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON){
+    SKVRowBuilder kvRowBuilder = {0};
+    int32_t code = tdInitKVRowBuilder(&kvRowBuilder);
+
+    if (TSDB_CODE_SUCCESS != code) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    if (pStmt->pVal->literal && strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
+      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal);
+    }
+
+    code = parseJsontoTagData(pStmt->pVal->literal, &kvRowBuilder, &pCxt->msgBuf, pSchema->colId);
+    if (TSDB_CODE_SUCCESS != code) {
+      return code;
+    }
+
+    SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+    if (NULL == row) {
+      tdDestroyKVRowBuilder(&kvRowBuilder);
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    pReq->nTagVal = kvRowLen(row);
+    pReq->pTagVal = row;
+    pStmt->pVal->datum.p = row;  // for free
+    tdDestroyKVRowBuilder(&kvRowBuilder);
+  }else{
+    pReq->nTagVal = pStmt->pVal->node.resType.bytes;
+    if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) {
+      pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE;
+    }
+    pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal);
   }
-  pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -4652,7 +4680,26 @@ static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) {
     return code;
   }
 
+  if (pStmt->dataType.type == TSDB_DATA_TYPE_JSON && pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+  }
+
+  if (pStmt->dataType.type == TSDB_DATA_TYPE_JSON && pStmt->alterType == TSDB_ALTER_TABLE_ADD_COLUMN) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COL_JSON);
+  }
+
+  if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType ==  TSDB_ALTER_TABLE_DROP_TAG) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "can not drop tag if there is only one tag");
+  }
+
   if (TSDB_SUPER_TABLE == pTableMeta->tableType) {
+    SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
+    if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON &&
+        (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG ||
+         pStmt->alterType ==  TSDB_ALTER_TABLE_DROP_TAG ||
+         pStmt->alterType ==  TSDB_ALTER_TABLE_UPDATE_TAG_BYTES)) {
+      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+    }
     return TSDB_CODE_SUCCESS;
   } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);

source/libs/parser/src/parUtil.c

@@ -171,6 +171,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "Window query not supported, since the result of subquery not include valid timestamp column";
     case TSDB_CODE_PAR_INVALID_DROP_COL:
       return "No columns can be dropped";
+    case TSDB_CODE_PAR_INVALID_COL_JSON:
+      return "Only tag can be json type";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:
@@ -328,7 +330,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
   // set json NULL data
   uint8_t jsonNULL = TSDB_DATA_TYPE_NULL;
   int     jsonIndex = startColId + 1;
-  if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) {
+  if (!json || strtrim((char*)json) == 0 ||strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) {
     tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
     return TSDB_CODE_SUCCESS;
   }
@@ -360,12 +362,12 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
       retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
       goto end;
     }
-    //    if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){
-    //      tscError("json key too long error");
-    //      retCode =  tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL);
-    //      goto end;
-    //    }
     size_t keyLen = strlen(jsonKey);
+    if(keyLen > TSDB_MAX_JSON_KEY_LEN){
+      qError("json key too long error");
+      retCode =  buildSyntaxErrMsg(pMsgBuf, "json key too long, more than 256", jsonKey);
+      goto end;
+    }
     if (keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL) {
       continue;
     }

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -1089,7 +1089,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
   }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV ||
              opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
     printf("1result:%f,except:%f\n", *((double *)colDataGetData(column, 0)), exceptValue);
-    ASSERT_TRUE(abs(*((double *)colDataGetData(column, 0)) - exceptValue) < 1e-15);
+    ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001);
   }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
     printf("2result:%ld,except:%f\n", *((int64_t *)colDataGetData(column, 0)), exceptValue);
     ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue);
@@ -1107,8 +1107,10 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
 
 TEST(columnTest, json_column_arith_op) {
   scltInitLogFile();
-  char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
+  char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
 
+  char rightv[256] = {0};
+  memcpy(rightv, rightvTmp, strlen(rightvTmp));
   SKVRowBuilder kvRowBuilder;
   tdInitKVRowBuilder(&kvRowBuilder);
   parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
@@ -1189,8 +1191,10 @@ void *prepareNchar(char* rightData){
 
 TEST(columnTest, json_column_logic_op) {
   scltInitLogFile();
-  char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
+  char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
 
+  char rightv[256] = {0};
+  memcpy(rightv, rightvTmp, strlen(rightvTmp));
   SKVRowBuilder kvRowBuilder;
   tdInitKVRowBuilder(&kvRowBuilder);
   parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);

source/util/src/tutil.c

@@ -52,7 +52,7 @@ size_t strtrim(char *z) {
   int32_t j = 0;
 
   int32_t delta = 0;
-  while (z[j] == ' ') {
+  while (isspace(z[j])) {
     ++j;
   }
 
@@ -65,9 +65,9 @@ size_t strtrim(char *z) {
 
   int32_t stop = 0;
   while (z[j] != 0) {
-    if (z[j] == ' ' && stop == 0) {
+    if (isspace(z[j]) && stop == 0) {
       stop = j;
-    } else if (z[j] != ' ' && stop != 0) {
+    } else if (!isspace(z[j]) && stop != 0) {
       stop = 0;
     }