diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8a87a1bd2eb3670e34ecdb24c711c2b0c904e9b1..2dd887ae7436784902db44c11ffec9bb80e19c51 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -608,7 +608,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) { +static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); @@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co pBuilder->nCols++; - int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; if (tlen > pBuilder->alloc - pBuilder->size) { while (tlen > pBuilder->alloc - pBuilder->size) { pBuilder->alloc *= 2; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 032cb441224c9e4f114b1f71b74ed4208f887286..11120211240540dd1e963fc1a5276e52941bef26 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor { int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; -extern tDataTypeDescriptor tDataTypes[15]; +extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; bool isValidDataType(int32_t type); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 60535e2f493a3bcb1769f067fb7e13a6fb715452..a11bb5162dc376c0bbdee407781f9ce324534607 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -599,6 +599,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST TAOS_DEF_ERROR_CODE(0, 0x2624) #define TSDB_CODE_PAR_INVALID_OPTION_UNIT TAOS_DEF_ERROR_CODE(0, 0x2625) #define TSDB_CODE_PAR_INVALID_KEEP_UNIT TAOS_DEF_ERROR_CODE(0, 0x2626) +#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2627) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tdef.h b/include/util/tdef.h index f276a1a8123cca46ea8f3252ee4550650866590f..0289bb0df7aefab223175a0dafd62021d40735ef 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -254,6 +254,14 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_MAX_JSON_TAG_LEN 16384 +#define TSDB_JSON_PLACEHOLDER 0x7F +#define TSDB_JSON_null 0x00 +#define TSDB_JSON_KEY_NULL 0x00 +#define TSDB_JSON_NOT_NULL 0x01 +#define TSDB_JSON_NULL 0x00 + + #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 diff --git a/include/util/tjson.h b/include/util/tjson.h index 6b2221f704c768e494ba797913ede2deb5f6a7f4..9f347e58967ee69995a57a629f4bcb237bacb7df 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -76,6 +76,7 @@ char* tjsonToString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson); SJson* tjsonParse(const char* pStr); +bool tjsonIsObject(const SJson* pJson); #ifdef __cplusplus } diff --git a/include/util/tutil.h b/include/util/tutil.h index e0f92be76a90d5abbfa22366d8513f3e06facbbd..444a893a888abb5c28de0c70476af9beee32cc10 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -26,8 +26,6 @@ extern "C" { #endif int32_t strdequote(char *src); -int32_t strndequote(char *dst, const char *z, int32_t len); -int32_t strRmquote(char *z, int32_t len); size_t strtrim(char *src); char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); char **strsplit(char *src, const char *delim, int32_t *num); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index ef75adeb5d69bdd03ca1bd724cdc7bb793364262..6af4777537fc8cb2538d861087d61f298f588778 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -365,7 +365,7 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i *maxIndex = 0; } -tDataTypeDescriptor tDataTypes[15] = { +tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, @@ -388,6 +388,7 @@ tDataTypeDescriptor tDataTypes[15] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, + {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, }; char tTokenTypeSwitcher[13] = { diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index a0503e43a187d8b136b7e8ab7673be6a15ebe230..a546d9bb6b120712cc24ac660db1fa1d1b12fc15 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) { } case TSDB_DATA_TYPE_BINARY: { pVar->pz = strndup(z, n); - pVar->nLen = strRmquote(pVar->pz, n); + //pVar->nLen = strRmquote(pVar->pz, n); break; } case TSDB_DATA_TYPE_TIMESTAMP: { diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 742ab303d3b419c46f344815c04ee7e2f379728c..460a089802a2629f38abbb56100d95298c6ac6f1 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -47,6 +47,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); +int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a14053ada7b184d072b05d0e104b46f8e28b402c..0dcf04ecc0f71a8910e2b5cdf996b43fb79d7070 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -47,6 +47,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { } static void trimEscape(SToken* pName) { + // todo need to deal with `ioo``ii` -> ioo`ii if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) { pName->z += 1; pName->n -= 2; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 435ab317e697dfc2f12b774572a24ee9c1a1288e..c9a0f4114e68cde442e769ca68f7998042c23d20 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -377,35 +377,15 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } - if (IS_NUMERIC_TYPE(type) && pToken->n == 0) { - return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z); - } - // Remove quotation marks if (TK_NK_STRING == pToken->type) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } - // delete escape character: \\, \', \" - char delim = pToken->z[0]; - int32_t cnt = 0; - int32_t j = 0; - for (uint32_t k = 1; k < pToken->n - 1; ++k) { - if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) { - tmpTokenBuf[j] = pToken->z[k + 1]; - cnt++; - j++; - k++; - continue; - } - tmpTokenBuf[j] = pToken->z[k]; - j++; - } - - tmpTokenBuf[j] = 0; + int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); pToken->z = tmpTokenBuf; - pToken->n -= 2 + cnt; + pToken->n = len; } return TSDB_CODE_SUCCESS; @@ -582,6 +562,13 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return func(pMsgBuf, pToken->z, pToken->n, param); } + case TSDB_DATA_TYPE_JSON: { + if(pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); + } + return func(pMsgBuf, pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_TIMESTAMP: { int64_t tmpVal; if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { @@ -702,8 +689,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + if(pColList->numOfCols > pColList->numOfBound){ + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + } return TSDB_CODE_SUCCESS; } @@ -720,22 +709,31 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi int8_t type = pa->schema->type; int16_t colId = pa->schema->colId; + if(TSDB_DATA_TYPE_JSON == type){ + return parseJsontoTagData(value, pa->builder, pMsgBuf, colId); + } + + if (value == NULL) { // it is a null data + // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); + return TSDB_CODE_SUCCESS; + } + if (TSDB_DATA_TYPE_BINARY == type) { STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); - tdAddColToKVRow(pa->builder, colId, type, pa->buf); + tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf)); } else if (TSDB_DATA_TYPE_NCHAR == type) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { char buf[512] = {0}; snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value);; + return buildSyntaxErrMsg(pMsgBuf, buf, value); } varDataSetLen(pa->buf, output); - tdAddColToKVRow(pa->builder, colId, type, pa->buf); + tdAddColToKVRow(pa->builder, colId, varDataTLen(pa->buf)); } else { - tdAddColToKVRow(pa->builder, colId, type, value); + tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b3947b872e2afa5dffecb40150213310abd7804a..bd461c762213cd8fd9adfc4245c15ac6d95501a7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1542,6 +1542,17 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS return TSDB_CODE_SUCCESS; } +static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) { + SNode* pNode; + FOREACH(pNode, pStmt->pTags) { + SColumnDefNode* pCol = (SColumnDefNode*)pNode; + if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){ + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) { if (NULL == pFuncs) { return TSDB_CODE_SUCCESS; @@ -1577,6 +1588,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY); } + if (TSDB_CODE_SUCCESS == code) { + code = checkTableTags(pCxt, pStmt); + } return code; } @@ -2893,11 +2907,19 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS } SVariant var; valueNodeToVariant(pVal, &var); + if(pSchema->type == TSDB_DATA_TYPE_JSON){ + if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz); + } + return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId); + } + char tagVal[TSDB_MAX_TAGS_LEN] = {0}; int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true); if (TSDB_CODE_SUCCESS == code) { - tdAddColToKVRow(pBuilder, pSchema->colId, pSchema->type, tagVal); + tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]); } + return code; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 85d574d9e7e4d79c9a26e8d32f3ea503b9e90e93..4ace62a2f15cac4a4fe534f3667329fa8e3a4339 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -14,6 +14,7 @@ */ #include "parUtil.h" +#include "cJSON.h" static char* getSyntaxErrFormat(int32_t errCode) { switch (errCode) { @@ -95,6 +96,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid option %s unit: %c, only m, h, d allowed"; case TSDB_CODE_PAR_INVALID_KEEP_UNIT: return "Invalid option keep unit: %c, %c, %c, only m, h, d allowed"; + case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG: + return "Only one tag if there is a json tag"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -195,24 +198,180 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) { } int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { - // delete escape character: \\, \', \" + if (len <=0 || dlen <= 0) return 0; + char delim = src[0]; - int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < len - 1; ++k) { if (j >= dlen) { - break; + dst[j - 1] = '\0'; + return j; } - if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) { + if (src[k] == delim && src[k + 1] == delim) { // deal with "", '' dst[j] = src[k + 1]; - cnt++; j++; k++; continue; } + + if (src[k] == '\\') { // deal with escape character + if(src[k+1] == 'n'){ + dst[j] = '\n'; + }else if(src[k+1] == 'r'){ + dst[j] = '\r'; + }else if(src[k+1] == 't'){ + dst[j] = '\t'; + }else if(src[k+1] == '\\'){ + dst[j] = '\\'; + }else if(src[k+1] == '\''){ + dst[j] = '\''; + }else if(src[k+1] == '"'){ + dst[j] = '"'; + }else if(src[k+1] == '%' || src[k+1] == '_'){ + dst[j++] = src[k]; + dst[j] = src[k+1]; + }else{ + dst[j] = src[k+1]; + } + j++; + k++; + continue; + } + dst[j] = src[k]; j++; } dst[j] = '\0'; return j; } + +static bool isValidateTag(char *input) { + if (!input) return false; + for (size_t i = 0; i < strlen(input); ++i) { + if (isprint(input[i]) == 0) return false; + } + return true; +} + +int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){ + // set json NULL data + uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL; + uint8_t jsonNULL = TSDB_JSON_NULL; + int jsonIndex = startColId + 1; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES); // add json null type + if (!json || strtrim(json) == 0 || strncasecmp(json, "null", 4) == 0){ + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); // add json null value + return TSDB_CODE_SUCCESS; + } + + // set json real data + cJSON *root = cJSON_Parse(json); + if (root == NULL){ + return buildSyntaxErrMsg(pMsgBuf, "json parse error", json); + } + + int size = cJSON_GetArraySize(root); + if(!cJSON_IsObject(root)){ + return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); + } + + int retCode = 0; + SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + for(int i = 0; i < size; i++) { + cJSON* item = cJSON_GetArrayItem(root, i); + if (!item) { + qError("json inner error:%d", i); + return buildSyntaxErrMsg(pMsgBuf, "json inner error", json); + goto end; + } + + char *jsonKey = item->string; + if(!isValidateTag(jsonKey)){ + retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); + goto end; + } +// if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){ +// tscError("json key too long error"); +// retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL); +// goto end; +// } + size_t keyLen = strlen(jsonKey); + if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){ + continue; + } + + if(taosHashGetSize(keyHash) == 0){ + uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type + } + // json key encode by binary + void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1); + if(!tagKey) { + retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + strncpy(varDataVal(tagKey), jsonKey, keyLen); + taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + + varDataSetLen(tagKey, keyLen); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey)); // add json key + taosMemoryFree(tagKey); + + if(item->type == cJSON_String){ // add json value format: type|data + char *jsonValue = item->valuestring; + int32_t valLen = (int32_t)strlen(jsonValue); + char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1); + if(!tagVal) { + retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } ; + tagVal[0] = TSDB_DATA_TYPE_NCHAR; + char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES); + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData), + (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { + qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); + retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); + taosMemoryFree(tagVal); + goto end; + } + + varDataSetLen(tagData, valLen); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData)); + taosMemoryFree(tagVal); + }else if(item->type == cJSON_Number){ + if(!isfinite(item->valuedouble)){ + qError("json value is invalidate"); + retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json); + goto end; + } + char tagVal[LONG_BYTES + CHAR_BYTES] = {0}; + tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT + : TSDB_DATA_TYPE_DOUBLE; + char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES); + if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble; + else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES); + }else if(item->type == cJSON_True || item->type == cJSON_False){ + char tagVal[CHAR_BYTES + CHAR_BYTES] = {0}; + tagVal[0] = TSDB_DATA_TYPE_BOOL; + tagVal[1] = (char)(item->valueint); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES); + }else if(item->type == cJSON_NULL){ + char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL}; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES); + } + else{ + retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json); + goto end; + } + } + + if(taosHashGetSize(keyHash) == 0){ // set json NULL true + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); + } + +end: + taosHashCleanup(keyHash); + cJSON_Delete(root); + return retCode; +} \ No newline at end of file diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 191143de1274f7e715f68e577e4bd2cae54449f4..711ece3b013d9ccd19f50f20e84da96011ae44dc 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -57,7 +57,7 @@ OptrStr gOptrStr[] = { {OP_TYPE_IS_NOT_UNKNOWN, "not unknown"}, // json operator - {OP_TYPE_JSON_GET_VALUE, "json get"}, + {OP_TYPE_JSON_GET_VALUE, "->"}, {OP_TYPE_JSON_CONTAINS, "json contains"} }; diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 93a843fee8ccbece4ac3e690e1905237b7a34485..8df404f85fd99043590357e8f4ce1c485223c19d 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -276,3 +276,4 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void } SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } +bool tjsonIsObject(const SJson* pJson) { return cJSON_IsObject(pJson); } \ No newline at end of file diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 8133e4d237f50c525ef653dd064854619e847dcc..adb6a37ba737fbf1ccafe35231853f8a2fc4cfc8 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -47,65 +47,6 @@ int32_t strdequote(char *z) { return j + 1; // only one quote, do nothing } -int32_t strRmquote(char *z, int32_t len) { - // delete escape character: \\, \', \" - char delim = z[0]; - if (delim != '\'' && delim != '\"') { - return len; - } - - int32_t cnt = 0; - int32_t j = 0; - for (uint32_t k = 1; k < len - 1; ++k) { - if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) { - if (z[k] == '\\' && z[k + 1] == '_') { - // match '_' self - } else { - z[j] = z[k + 1]; - cnt++; - j++; - k++; - continue; - } - } - - z[j] = z[k]; - j++; - } - - z[j] = 0; - - return len - 2 - cnt; -} - -int32_t strndequote(char *dst, const char *z, int32_t len) { - assert(dst != NULL); - if (z == NULL || len == 0) { - return 0; - } - - int32_t quote = z[0]; - int32_t i = 1, j = 0; - - while (z[i] != 0) { - if (z[i] == quote) { - if (z[i + 1] == quote) { - dst[j++] = (char)quote; - i++; - } else { - dst[j++] = 0; - return (j - 1); - } - } else { - dst[j++] = z[i]; - } - - i++; - } - - return j + 1; // only one quote, do nothing -} - size_t strtrim(char *z) { int32_t i = 0; int32_t j = 0; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index b89d517ad3e4396d1472e0b0a6a8a7d2333ab7c5..5a96a2a87dc8655ebbb305e10264889729b62f6d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -226,63 +226,27 @@ int32_t shellRunCommand(TAOS *con, char *command) { } } - bool esc = false; - char quote = 0, *cmd = command, *p = command; + char quote = 0, *cmd = command; for (char c = *command++; c != 0; c = *command++) { - if (esc) { - switch (c) { - case 'n': - c = '\n'; - break; - case 'r': - c = '\r'; - break; - case 't': - c = '\t'; - break; - case 'G': - *p++ = '\\'; - break; - case '\'': - case '"': - if (quote) { - *p++ = '\\'; - } - break; - } - *p++ = c; - esc = false; + if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) { + command ++; continue; } - if (c == '\\') { - if (quote != 0 && (*command == '_' || *command == '\\')) { - // DO nothing - } else { - esc = true; - continue; - } - } - if (quote == c) { quote = 0; - } else if (quote == 0 && (c == '\'' || c == '"')) { + } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) { quote = c; - } - - *p++ = c; - if (c == ';' && quote == 0) { - c = *p; - *p = 0; + } else if (c == ';' && quote == 0) { + c = *command; + *command = 0; if (shellRunSingleCommand(con, cmd) < 0) { return -1; } - *p = c; - p = cmd; + *command = c; + cmd = command; } } - - *p = 0; return shellRunSingleCommand(con, cmd); } @@ -573,19 +537,23 @@ static void shellPrintNChar(const char *str, int length, int width) { while (pos < length) { TdWchar wc; int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - if (bytes == 0) { + if (bytes <= 0) { break; } - pos += bytes; - if (pos > length) { + if (pos + bytes > length) { break; } - + int w = 0; #ifdef WINDOWS - int w = bytes; + w = bytes; #else - int w = taosWcharWidth(wc); + if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){ + w = bytes; + }else{ + w = taosWcharWidth(wc); + } #endif + pos += bytes; if (w <= 0) { continue; } @@ -679,6 +647,7 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: