提交 10036749 编写于 作者: wmmhello's avatar wmmhello

revert get json type in qid return

上级 bb373c96
......@@ -80,7 +80,6 @@ typedef struct SInternalField {
TAOS_FIELD field;
bool visible;
SExprInfo *pExpr;
TAOS_FIELD fieldJson; // for tag json
} SInternalField;
typedef struct SParamInfo {
......
......@@ -1910,29 +1910,20 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
char* oriAliasName1 = NULL;
char oriAliasName2[TSDB_COL_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_MAX_JSON_KEY_LEN + 4 + 1] = {0};
if (pSchema->type == TSDB_DATA_TYPE_JSON && pItem->pNode->tokenId == TK_ARROW) {
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN + 1] = {0};
if (pItem->aliasName){
jsonKeyMd5(pItem->aliasName, strlen(pItem->aliasName), keyMd5);
oriAliasName1 = pItem->aliasName;
}else{
jsonKeyMd5(pItem->pNode->exprToken.z, pItem->pNode->exprToken.n, keyMd5);
tstrncpy(oriAliasName2, pItem->pNode->exprToken.z,
pItem->pNode->exprToken.n + 1 < sizeof(oriAliasName2) ? pItem->pNode->exprToken.n + 1 : sizeof(oriAliasName2));
}
tstrncpy(pExpr->base.aliasName, keyMd5, sizeof(pExpr->base.aliasName));
} else {
if (pItem->aliasName){
tstrncpy(pExpr->base.aliasName, pItem->aliasName, sizeof(pExpr->base.aliasName));
oriAliasName1 = pItem->aliasName;
}else{
tstrncpy(pExpr->base.aliasName, pSchema->name, sizeof(pExpr->base.aliasName));
oriAliasName1 = pSchema->name;
tstrncpy(pExpr->base.aliasName, pItem->pNode->exprToken.z,
pItem->pNode->exprToken.n + 1 < sizeof(pExpr->base.aliasName) ? pItem->pNode->exprToken.n + 1 : sizeof(pExpr->base.aliasName));
}
}
char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName;
tstrncpy(pExpr->base.aliasName, colName, sizeof(pExpr->base.aliasName));
}else{
char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName;
tstrncpy(pExpr->base.aliasName, colName, sizeof(pExpr->base.aliasName));
}
SColumnList ids = {0};
ids.num = 1;
ids.ids[0] = *pIndex;
......@@ -1942,7 +1933,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
ids.num = 0;
}
insertResultField(pQueryInfo, startPos, &ids, pExpr->base.resBytes, (int8_t)pExpr->base.resType, oriAliasName1 ? oriAliasName1 : oriAliasName2, pExpr);
insertResultField(pQueryInfo, startPos, &ids, pExpr->base.resBytes, (int8_t)pExpr->base.resType, pExpr->base.aliasName, pExpr);
}
static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) {
......@@ -2195,12 +2186,6 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi
TAOS_FIELD f = tscCreateField(type, fieldName, bytes);
SInternalField* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f);
if (type == TSDB_DATA_TYPE_JSON){
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN + 1] = {0};
jsonKeyMd5(fieldName, strlen(fieldName), keyMd5);
strncpy(pInfo->fieldJson.name, keyMd5, sizeof(pInfo->fieldJson.name));
pInfo->fieldJson.type = TSDB_DATA_TYPE_JSON;
}
pInfo->pExpr = pSqlExpr;
return TSDB_CODE_SUCCESS;
......@@ -7527,20 +7512,15 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo
insertResultField(pQueryInfo, pos, &ids, s->bytes, (int8_t)s->type, pColIndex->name, pExpr);
pExpr->base.colInfo.flag = TSDB_COL_TAG;
memset(pExpr->base.aliasName, 0, sizeof(pExpr->base.aliasName));
tstrncpy(pExpr->base.aliasName, pColIndex->name, sizeof(pExpr->base.aliasName));
tstrncpy(pExpr->base.token, pColIndex->name, sizeof(pExpr->base.token));
if(s->type == TSDB_DATA_TYPE_JSON){
SStrToken t0 = {.z = pColIndex->name};
getJsonKey(&t0);
tVariantCreateFromBinary(&(pExpr->base.param[pExpr->base.numOfParams]), t0.z,
t0.n, TSDB_DATA_TYPE_BINARY);
pExpr->base.numOfParams++;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN + 1] = {0};
jsonKeyMd5(pColIndex->name, strlen(pColIndex->name), keyMd5);
tstrncpy(pExpr->base.aliasName, keyMd5, sizeof(pExpr->base.aliasName));
tstrncpy(pExpr->base.token, keyMd5, sizeof(pExpr->base.token));
tstrncpy(pColIndex->name, t0.z, t0.n + 1);
}else {
tstrncpy(pExpr->base.aliasName, s->name, sizeof(pExpr->base.aliasName));
tstrncpy(pExpr->base.token, s->name, sizeof(pExpr->base.aliasName));
}
} else {
// if this query is "group by" normal column, time window query is not allowed
......
......@@ -855,7 +855,6 @@ static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo,
pSqlExpr->numOfParams = htons(pExpr->numOfParams);
pSqlExpr->resColId = htons(pExpr->resColId);
pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters);
tstrncpy(pSqlExpr->aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN);
(*pMsg) += sizeof(SSqlExpr);
for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log
......@@ -2644,38 +2643,12 @@ int tscProcessShowCreateRsp(SSqlObj *pSql) {
return tscLocalResultCommonBuilder(pSql, 1);
}
static void updateFieldForJson(SSqlObj *pSql, SQueryTableRsp *pQueryAttr){
if(pQueryAttr->tJsonSchLen <= 0) {
return;
}
SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd);
SFieldInfo *pFieldInfo = &pQueryInfo->fieldsInfo;
for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) {
SInternalField *pField = tscFieldInfoGetInternalField(pFieldInfo, i);
if (pField->field.type == TSDB_DATA_TYPE_JSON) {
for (int k = 0; k < pQueryAttr->tJsonSchLen; ++k) {
if (strncmp(pField->fieldJson.name, pQueryAttr->tagJsonSchema[k].name, TSDB_MAX_JSON_KEY_LEN) == 0
&& pQueryAttr->tagJsonSchema[k].type != TSDB_DATA_TYPE_JSON) {
pField->fieldJson.type = pQueryAttr->tagJsonSchema[k].type;
pField->fieldJson.bytes = TYPE_BYTES[pField->fieldJson.type];
tscDebug("0x%" PRIx64 " change json type %s:%s to %d", pSql->self, pField->field.name, pQueryAttr->tagJsonSchema[k].name,
pField->fieldJson.type);
break;
}
}
}
}
}
int tscProcessQueryRsp(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res;
SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp;
pQueryAttr->qId = htobe64(pQueryAttr->qId);
pQueryAttr->tJsonSchLen = htons(pQueryAttr->tJsonSchLen);
updateFieldForJson(pSql, pQueryAttr);
pRes->qId = pQueryAttr->qId;
pRes->data = NULL;
......
......@@ -439,12 +439,6 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
if (pField->visible) {
f[j] = pField->field;
if(f[j].type == TSDB_DATA_TYPE_JSON){
f[j].type = pField->fieldJson.type;
if(!IS_VAR_DATA_TYPE(f[j].type) && f[j].type != TSDB_DATA_TYPE_JSON){
f[j].bytes = pField->fieldJson.bytes;
}
}
// revise the length for binary and nchar fields
if (f[j].type == TSDB_DATA_TYPE_BINARY) {
f[j].bytes -= VARSTR_HEADER_SIZE;
......
......@@ -3686,10 +3686,6 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql) {
int32_t type = pInfo->field.type;
int32_t bytes = pInfo->field.bytes;
if(type == TSDB_DATA_TYPE_JSON && pInfo->fieldJson.type != TSDB_DATA_TYPE_JSON){
type = pInfo->fieldJson.type;
bytes = pInfo->fieldJson.bytes;
}
if (!IS_VAR_DATA_TYPE(type) && type != TSDB_DATA_TYPE_JSON) {
pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : pRes->urow[i];
......
......@@ -701,45 +701,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bool convertNchar) {
// generated the user-defined column result
if (pInfo->field.type == TSDB_DATA_TYPE_JSON){
char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
if (buffer == NULL) return;
pRes->buffer[i] = buffer;
// string terminated char for binary data
memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows);
char* p = pRes->urow[i];
int32_t offset = 0;
for (int32_t k = 0; k < pRes->numOfRows; ++k) {
char* dst = pRes->buffer[i] + k * offset;
char* realData = p + CHAR_BYTES;
char type = *p;
if (type == TSDB_DATA_TYPE_JSON && isNull(realData, TSDB_DATA_TYPE_JSON)) {
type = pInfo->fieldJson.type;
setNull(realData, type, 0);
}
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON) {
if(convertNchar && !isNull(realData, TSDB_DATA_TYPE_NCHAR)){
int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(dst));
varDataSetLen(dst, length);
if (length == 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)varDataVal(realData));
}
}else{
memcpy(dst, realData, varDataTLen(realData));
}
offset = pInfo->field.bytes;
}else {
assert(type <= TSDB_DATA_TYPE_DOUBLE && type >=TSDB_DATA_TYPE_BOOL);
memcpy(dst, realData, tDataTypes[(int32_t)type].bytes);
offset = tDataTypes[(int32_t)type].bytes;
}
p += pInfo->field.bytes;
}
memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
}else if (pInfo->pExpr->pExpr == NULL && TSDB_COL_IS_UD_COL(pInfo->pExpr->base.colInfo.flag)) {
if (pInfo->pExpr->pExpr == NULL && TSDB_COL_IS_UD_COL(pInfo->pExpr->base.colInfo.flag)) {
if (pInfo->pExpr->base.param[0].nType == TSDB_DATA_TYPE_NULL) {
setNullN(pRes->urow[i], pInfo->field.type, pInfo->field.bytes, (int32_t) pRes->numOfRows);
} else {
......@@ -788,6 +750,50 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
p += pInfo->field.bytes;
}
memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
}else if (pInfo->field.type == TSDB_DATA_TYPE_JSON) {
// convert unicode to native code in a temporary buffer extra one byte for terminated symbol
char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
if (buffer == NULL) return;
pRes->buffer[i] = buffer;
// string terminated char for binary data
memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows);
char* p = pRes->urow[i];
for (int32_t k = 0; k < pRes->numOfRows; ++k) {
char* dst = pRes->buffer[i] + k * pInfo->field.bytes;
char type = *p;
char* realData = p + CHAR_BYTES;
if (type == TSDB_DATA_TYPE_NCHAR && isNull(realData, TSDB_DATA_TYPE_NCHAR)) {
memcpy(dst, realData, varDataTLen(realData));
} else if (type == TSDB_DATA_TYPE_BINARY) {
assert(*(uint32_t*)varDataVal(realData) == TSDB_DATA_JSON_null); // json null value
assert(varDataLen(realData) == INT_BYTES);
sprintf(varDataVal(dst), "%s", "null");
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if (type == TSDB_DATA_TYPE_NCHAR) {
int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(dst));
varDataSetLen(dst, length);
if (length == 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p);
}
}else if (type == TSDB_DATA_TYPE_DOUBLE) {
double jsonVd = *(double*)(realData);
sprintf(varDataVal(dst), "%.9lf", jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if (type == TSDB_DATA_TYPE_BIGINT) {
int64_t jsonVd = *(int64_t*)(realData);
sprintf(varDataVal(dst), "%" PRId64, jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if (type == TSDB_DATA_TYPE_BOOL) {
sprintf(varDataVal(dst), "%s", (*((char *)realData) == 1) ? "true" : "false");
varDataSetLen(dst, strlen(varDataVal(dst)));
}else {
assert(0);
}
p += pInfo->field.bytes;
}
memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
}
if (convertNchar) {
......
......@@ -28,7 +28,7 @@ typedef void* qinfo_t;
* @param qinfo
* @return
*/
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId, void** tJsonSchema);
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId);
/**
......
......@@ -227,7 +227,7 @@ do { \
#define TSDB_MAX_JSON_TAGS_LEN (4096*TSDB_NCHAR_SIZE + 2 + 1) // 2->var_header_len 1->type
#define TSDB_MAX_TAGS 128
#define TSDB_MAX_TAG_CONDITIONS 1024
#define TSDB_MAX_JSON_KEY_LEN 65
#define TSDB_MAX_JSON_KEY_LEN 256
#define TSDB_MAX_JSON_KEY_MD5_LEN 16
#define TSDB_AUTH_LEN 16
......
......@@ -258,11 +258,6 @@ typedef struct SSchema {
int16_t bytes;
} SSchema;
typedef struct TagJsonSSchema {
uint8_t type;
char name[TSDB_MAX_JSON_KEY_LEN];
} TagJsonSSchema;
typedef struct {
int32_t contLen;
int32_t vgId;
......@@ -519,8 +514,6 @@ typedef struct {
typedef struct {
int32_t code;
union{uint64_t qhandle; uint64_t qId;}; // query handle
uint16_t tJsonSchLen;
TagJsonSSchema tagJsonSchema[];
} SQueryTableRsp;
// todo: the show handle should be replaced with id
......
......@@ -419,8 +419,6 @@ int tsdbCompact(STsdbRepo *pRepo);
bool tsdbNoProblem(STsdbRepo* pRepo);
// unit of walSize: MB
int tsdbCheckWal(STsdbRepo *pRepo, uint32_t walSize);
// for tag json
uint8_t getTagJsonType(STsdbRepo* tsdb, uint64_t uid, char* key, int32_t len);
#ifdef __cplusplus
}
......
......@@ -67,39 +67,7 @@ void freeParam(SQueryParam *param) {
tfree(param->prevResult);
}
static void* setJsonTagSchema(void* tsdb, int16_t numOfOutput, SExprInfo *pExprs, SArray* pTableIdList){
uint16_t cnt = 0;
for (int i = 0; i < numOfOutput; ++i) {
SSqlExpr* sqlExpr = &pExprs[i].base;
if (sqlExpr->colType == TSDB_DATA_TYPE_JSON && sqlExpr->numOfParams > 0) {
cnt ++;
}
}
if(cnt <= 0) return NULL;
void* tJsonSchData = calloc(1, SHORT_BYTES + cnt*sizeof(TagJsonSSchema));
*(uint16_t*)(tJsonSchData) = cnt;
void* tmp = tJsonSchData + SHORT_BYTES;
for (int i = 0; i < numOfOutput; ++i) {
SSqlExpr* sqlExpr = &pExprs[i].base;
if (sqlExpr->colType == TSDB_DATA_TYPE_JSON && sqlExpr->numOfParams > 0){
TagJsonSSchema* schema = (TagJsonSSchema*)(tmp);
schema->type = TSDB_DATA_TYPE_JSON;
tstrncpy(schema->name, sqlExpr->aliasName, TSDB_MAX_JSON_KEY_LEN);
for (int j = 0; j < taosArrayGetSize(pTableIdList); ++j) {
STableIdInfo *id = taosArrayGet(pTableIdList, j);
uint8_t type = getTagJsonType(tsdb, id->uid, sqlExpr->param[0].pz, sqlExpr->param[0].nLen);
if(type != TSDB_DATA_TYPE_JSON) {
schema->type = type;
break;
}
}
tmp += sizeof(TagJsonSSchema);
}
}
return tJsonSchData;
}
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t qId, void** tJsonSchema) {
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t qId) {
assert(pQueryMsg != NULL && tsdb != NULL);
int32_t code = TSDB_CODE_SUCCESS;
......@@ -200,7 +168,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
assert(pQueryMsg->stableQuery == isSTableQuery);
(*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo);
*tJsonSchema = setJsonTagSchema(tsdb, pQueryMsg->numOfOutput, param.pExprs, param.pTableIdList);
param.sql = NULL;
param.pExprs = NULL;
param.pSecExprs = NULL;
......
......@@ -4112,16 +4112,5 @@ static int32_t tsdbQueryTableList(STable* pTable, SArray* pRes, void* filterInfo
return TSDB_CODE_SUCCESS;
}
uint8_t getTagJsonType(STsdbRepo* tsdb, uint64_t uid, char* key, int32_t len){
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
void* result = getJsonTagValueElment(pTable, key, len, NULL, TSDB_MAX_JSON_TAGS_LEN);
if(result){
return *(char*)result;
}else{
return TSDB_DATA_TYPE_JSON;
}
}
......@@ -239,25 +239,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
if (contLen != 0) {
qinfo_t pQInfo = NULL;
uint64_t qId = genQueryId();
void* tJsonSchema = NULL;
code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, qId, &tJsonSchema);
code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, qId);
int extSize = 0;
if (tJsonSchema != NULL){
uint16_t cnt = *(uint16_t*)tJsonSchema;
extSize = cnt * sizeof(TagJsonSSchema);
}
SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp) + extSize);
SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
pRsp->code = code;
pRsp->qId = 0;
pRet->len = sizeof(SQueryTableRsp) + extSize;
pRet->len = sizeof(SQueryTableRsp);
pRet->rsp = pRsp;
if (tJsonSchema != NULL){
pRsp->tJsonSchLen = htons(*(uint16_t*)tJsonSchema);
memcpy(pRsp->tagJsonSchema, tJsonSchema + SHORT_BYTES, extSize);
tfree(tJsonSchema);
}
int32_t vgId = pVnode->vgId;
// current connect is broken
......
......@@ -194,7 +194,7 @@ class TDTestCase:
tdSql.checkData(0, 0, "{\"k1\":\"\",\"k2\":true,\"k3\":false,\"k4\":55}")
tdSql.query("select jtag->'k2' from db_json_tag_test.jsons1_10")
tdSql.checkData(0, 0, True)
tdSql.checkData(0, 0, "true")
tdSql.query("select jtag from db_json_tag_test.jsons1 where jtag->'k1'=''")
tdSql.checkRows(1)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册