diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9868c2cc0d1fc9f13fe440438b040c257d089806..e47944c724132b5b0e23f82ed15e5373d71e589c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -638,6 +638,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x264F) #define TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY TAOS_DEF_ERROR_CODE(0, 0x2650) #define TSDB_CODE_PAR_INVALID_DROP_COL TAOS_DEF_ERROR_CODE(0, 0x2651) +#define TSDB_CODE_PAR_INVALID_COL_JSON TAOS_DEF_ERROR_CODE(0, 0x2652) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tdef.h b/include/util/tdef.h index 5cc687d7ab141c0eedabfcf6331f56af1a6175e5..4168a240033e4f251649cc31ad9f23e9ff37e725 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -233,6 +233,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_MAX_JSON_TAG_LEN 16384 +#define TSDB_MAX_JSON_KEY_LEN 256 #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3872d4df8ade6a53f5547d7f43390c61966e44db..eddf126aecb4a71c4b6867e666d6ee34e66f5111 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -677,7 +677,6 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD varDataSetLen(pVal->datum.p, output); break; } - case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); @@ -2477,6 +2476,9 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SN code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN); } } + if (TSDB_CODE_SUCCESS == code && pCol->dataType.type == TSDB_DATA_TYPE_JSON) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COL_JSON); + } int32_t len = strlen(pCol->colName); if (TSDB_CODE_SUCCESS == code && NULL != taosHashGet(pHash, pCol->colName, len)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); @@ -2484,7 +2486,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SN if (TSDB_CODE_SUCCESS == code) { if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN)) { - code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } } if (TSDB_CODE_SUCCESS == code) { @@ -4452,11 +4454,38 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS } pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); - pReq->nTagVal = pStmt->pVal->node.resType.bytes; - if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) { - pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE; + if(pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON){ + SKVRowBuilder kvRowBuilder = {0}; + int32_t code = tdInitKVRowBuilder(&kvRowBuilder); + + if (TSDB_CODE_SUCCESS != code) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (pStmt->pVal->literal && strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal); + } + + code = parseJsontoTagData(pStmt->pVal->literal, &kvRowBuilder, &pCxt->msgBuf, pSchema->colId); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + if (NULL == row) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return TSDB_CODE_OUT_OF_MEMORY; + } + pReq->nTagVal = kvRowLen(row); + pReq->pTagVal = row; + pStmt->pVal->datum.p = row; // for free + tdDestroyKVRowBuilder(&kvRowBuilder); + }else{ + pReq->nTagVal = pStmt->pVal->node.resType.bytes; + if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) { + pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE; + } + pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal); } - pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal); return TSDB_CODE_SUCCESS; } @@ -4653,6 +4682,25 @@ static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { return code; } + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON && + (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG || + pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG || + pStmt->alterType == TSDB_ALTER_TABLE_UPDATE_TAG_BYTES)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); + } + if (pStmt->dataType.type == TSDB_DATA_TYPE_JSON && pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); + } + + if (pStmt->dataType.type == TSDB_DATA_TYPE_JSON && pStmt->alterType == TSDB_ALTER_TABLE_ADD_COLUMN) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COL_JSON); + } + + if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "can not drop tag if there is only one tag"); + } + if (TSDB_SUPER_TABLE == pTableMeta->tableType) { return TSDB_CODE_SUCCESS; } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 3df9c8abf7f36d5f04b9ce79070c71dd63a712ff..85c84af05b9d0304f4967d7cbe381e1721663eb8 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -171,6 +171,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Window query not supported, since the result of subquery not include valid timestamp column"; case TSDB_CODE_PAR_INVALID_DROP_COL: return "No columns can be dropped"; + case TSDB_CODE_PAR_INVALID_COL_JSON: + return "Only tag can be json type"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -360,12 +362,12 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); goto end; } - // if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){ - // tscError("json key too long error"); - // retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL); - // goto end; - // } size_t keyLen = strlen(jsonKey); + if(keyLen > TSDB_MAX_JSON_KEY_LEN){ + qError("json key too long error"); + retCode = buildSyntaxErrMsg(pMsgBuf, "json key too long, more than 256", jsonKey); + goto end; + } if (keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL) { continue; } diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 714183157ff3ab95221ee90bc9bedac332f69673..216fceba3be0dd153d2d00e4dc5f1b1a41aa3656 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -35,8 +35,8 @@ class TDTestCase: def run(self): tdSql.prepare() print("============== STEP 1 ===== prepare data & validate json string") - # tdSql.error("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json, tagint int)") - # tdSql.error("create table if not exists jsons1(ts timestamp, data json) tags(tagint int)") + tdSql.error("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json, tagint int)") + tdSql.error("create table if not exists jsons1(ts timestamp, data json) tags(tagint int)") tdSql.execute("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") tdSql.execute("insert into jsons1_1 using jsons1 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')") tdSql.execute("insert into jsons1_2 using jsons1 tags('{\"tag1\":5,\"tag2\":\"beijing\"}') values (1591060628000, 2, true, 'json2', 'sss')") @@ -47,58 +47,62 @@ class TDTestCase: tdSql.execute("insert into jsons1_7 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") # test duplicate key using the first one. elimate empty key - #tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") + tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") #tdSql.query("select jtag from jsons1_8") #tdSql.checkData(0, 0, '{"tag1":null,"1tag$":2," ":90}') - # # test empty json string, save as jtag is NULL - # tdSql.execute("insert into jsons1_9 using jsons1 tags('\t') values (1591062328000, 24, NULL, '你就会', '2sdw')") - # tdSql.execute("CREATE TABLE if not exists jsons1_10 using jsons1 tags('')") - # tdSql.execute("CREATE TABLE if not exists jsons1_11 using jsons1 tags(' ')") - # tdSql.execute("CREATE TABLE if not exists jsons1_12 using jsons1 tags('{}')") - # tdSql.execute("CREATE TABLE if not exists jsons1_13 using jsons1 tags('null')") - # - # # test invalidate json - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('\"efwewf\"')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('3333')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('33.33')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('false')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('[1,true]')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{222}')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"fe\"}')") - # - # # test invalidate json key, key must can be printed assic char - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":[1,true]}')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":{}}')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"。loc\":\"fff\"}')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"\t\":\"fff\"}')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"试试\":\"fff\"}')") - # - # # test invalidate json value, value number can not be inf,nan TD-12166 - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":1.8e308}')") - # tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":-1.8e308}')") - # - # #test length limit - # char1= ''.join(['abcd']*64) - # char3= ''.join(['abcd']*1022) - # print(len(char3)) # 4088 - # tdSql.error("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s1\":5}')" % char1) # len(key)=257 - # tdSql.execute("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s\":5}')" % char1) # len(key)=256 - # tdSql.error("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TS\":\"%s\"}')" % char3) # len(object)=4097 - # tdSql.execute("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"T\":\"%s\"}')" % char3) # len(object)=4096 - # tdSql.execute("drop table if exists jsons1_15") - # tdSql.execute("drop table if exists jsons1_16") - # - # print("============== STEP 2 ===== alter table json tag") - # tdSql.error("ALTER STABLE jsons1 add tag tag2 nchar(20)") - # tdSql.error("ALTER STABLE jsons1 drop tag jtag") - # tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)") - # - # tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'") + # test empty json string, save as jtag is NULL + tdSql.execute("insert into jsons1_9 using jsons1 tags('\t') values (1591062328000, 24, NULL, '你就会', '2sdw')") + tdSql.execute("CREATE TABLE if not exists jsons1_10 using jsons1 tags('')") + tdSql.execute("CREATE TABLE if not exists jsons1_11 using jsons1 tags(' ')") + tdSql.execute("CREATE TABLE if not exists jsons1_12 using jsons1 tags('{}')") + tdSql.execute("CREATE TABLE if not exists jsons1_13 using jsons1 tags('null')") + + # test invalidate json + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('\"efwewf\"')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('3333')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('33.33')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('false')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('[1,true]')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{222}')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"fe\"}')") + # + # test invalidate json key, key must can be printed assic char + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":[1,true]}')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":{}}')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"。loc\":\"fff\"}')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"\t\":\"fff\"}')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"试试\":\"fff\"}')") + + # test invalidate json value, value number can not be inf,nan TD-12166 + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":1.8e308}')") + tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":-1.8e308}')") + # + #test length limit + char1= ''.join(['abcd']*64) + char3= ''.join(['abcd']*1021) + print(len(char3)) # 4084 + tdSql.error("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s1\":5}')" % char1) # len(key)=257 + tdSql.execute("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s\":5}')" % char1) # len(key)=256 + tdSql.error("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSSS\":\"%s\"}')" % char3) # len(object)=4096 + tdSql.execute("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSS\":\"%s\"}')" % char3) # len(object)=4095 + tdSql.execute("drop table if exists jsons1_15") + tdSql.execute("drop table if exists jsons1_16") + # + print("============== STEP 2 ===== alter table json tag") + tdSql.error("ALTER STABLE jsons1 add tag tag2 nchar(20)") + tdSql.error("ALTER STABLE jsons1 drop tag jtag") + tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)") + # + tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'") # tdSql.query("select jtag from jsons1_1") # tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') - # tdSql.execute("ALTER TABLE jsons1 CHANGE TAG jtag jtag_new") - # tdSql.execute("ALTER TABLE jsons1 CHANGE TAG jtag_new jtag") + tdSql.execute("ALTER TABLE jsons1 rename TAG jtag jtag_new") + tdSql.execute("ALTER TABLE jsons1 rename TAG jtag_new jtag") + + tdSql.execute("create table st(ts timestamp, i int) tags(t int)") + tdSql.error("ALTER STABLE st add tag jtag json") + tdSql.error("ALTER STABLE st add column jtag json") # # print("============== STEP 3 ===== query table") # # test error syntax