From 96b24d70643b8e8e54815bba816f827b040dd936 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 1 May 2020 16:34:02 +0800 Subject: [PATCH] [td-186] merge develop branch --- src/client/src/tscUtil.c | 6 ++++-- src/common/inc/tdataformat.h | 2 +- src/query/inc/qsqlparser.h | 2 +- src/query/inc/sql.y | 2 +- src/query/src/qparserImpl.c | 13 ++++++++----- src/query/src/sql.c | 2 +- src/rpc/src/rpcMain.c | 25 +++++++++++++++++-------- src/tsdb/src/tsdbRead.c | 7 ++++--- src/util/src/tcache.c | 2 +- src/util/tests/CMakeLists.txt | 3 +-- tests/examples/c/demo.c | 6 +++--- 11 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index fe24474b5b..dd654cdbd9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -603,14 +603,16 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); pDataBlock += sizeof(SSubmitBlk); - int32_t flen = 0; + int32_t flen = 0; // original total length of row for (int32_t i = 0; i < tinfo.numOfColumns; ++i) { flen += TYPE_BYTES[pSchema[i].type]; } char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); pBlock->len = 0; - for (int32_t i = 0; i < htons(pBlock->numOfRows); ++i) { + int32_t numOfRows = htons(pBlock->numOfRows); + + for (int32_t i = 0; i < numOfRows; ++i) { SDataRow trow = (SDataRow)pDataBlock; dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen); diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 489635420a..53cf2a1158 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -182,7 +182,7 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); -void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); +void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows); diff --git a/src/query/inc/qsqlparser.h b/src/query/inc/qsqlparser.h index 42dda2308f..08d4186291 100644 --- a/src/query/inc/qsqlparser.h +++ b/src/query/inc/qsqlparser.h @@ -277,7 +277,7 @@ SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); -void setCreatedMeterName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists); +void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists); void SQLInfoDestroy(SSqlInfo *pInfo); diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index c68c6b507a..55007597e2 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -264,7 +264,7 @@ signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} ////////////////////////////////// The CREATE TABLE statement /////////////////////////////// cmd ::= CREATE TABLE ifnotexists(Y) ids(X) cpxName(Z) create_table_args. { X.n += Z.n; - setCreatedMeterName(pInfo, &X, &Y); + setCreatedTableName(pInfo, &X, &Y); } %type create_table_args{SCreateTableSQL*} diff --git a/src/query/src/qparserImpl.c b/src/query/src/qparserImpl.c index 075dbc9d14..80e22d4514 100644 --- a/src/query/src/qparserImpl.c +++ b/src/query/src/qparserImpl.c @@ -463,7 +463,8 @@ int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); - /* truncate the column name */ + + // truncate the column name if (pName->n >= maxLen) { pName->n = maxLen - 1; } @@ -478,7 +479,9 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { pField->type = -1; - for (int8_t i = 0; i < sizeof(tDataTypeDesc) / sizeof(tDataTypeDesc[0]); ++i) { + int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar. + + for (int8_t i = 0; i < tListLen(tDataTypeDesc); ++i) { if ((strncasecmp(type->z, tDataTypeDesc[i].aName, tDataTypeDesc[i].nameLen) == 0) && (type->n == tDataTypeDesc[i].nameLen)) { pField->type = i; @@ -490,10 +493,10 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { * number of bytes in UCS-4 format, which is 4 times larger than the * number of characters */ - pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE; + pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR; } else if (i == TSDB_DATA_TYPE_BINARY) { /* for binary, the TOKENTYPE is the length of binary */ - pField->bytes = -(int32_t)type->type; + pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR; } break; } @@ -731,7 +734,7 @@ SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) return pQueryInfo; } -void setCreatedMeterName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists) { +void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists) { pInfo->pCreateTableInfo->name = *pMeterName; pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); } diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 08a8d41c69..d15905ce2a 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -2267,7 +2267,7 @@ static void yy_reduce( case 103: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - setCreatedMeterName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0); + setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0); } break; case 104: /* create_table_args ::= LP columnlist RP */ diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index dfb549b3f9..adada75d09 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -32,6 +32,8 @@ #include "rpcCache.h" #include "rpcTcp.h" #include "rpcHead.h" +#include "shash.h" + #define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) #define rpcHeadFromCont(cont) ((SRpcHead *) (cont - sizeof(SRpcHead))) @@ -260,7 +262,9 @@ void *rpcOpen(const SRpcInit *pInit) { } if (pRpc->connType == TAOS_CONN_SERVER) { - pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); + pRpc->hash = taosInitStrHash(pRpc->sessions, sizeof(pRpc), taosHashString); + +// pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); if (pRpc->hash == NULL) { tError("%s failed to init string hash", pRpc->label); rpcClose(pRpc); @@ -543,9 +547,10 @@ static void rpcCloseConn(void *thandle) { if ( pRpc->connType == TAOS_CONN_SERVER) { char hashstr[40] = {0}; - size_t size = sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); - taosHashRemove(pRpc->hash, hashstr, size); - + /*size_t size = */sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); +// taosHashRemove(pRpc->hash, hashstr, size); + taosDeleteStrHash(pRpc->hash, hashstr); + rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg pConn->pRspMsg = NULL; pConn->inType = 0; @@ -593,10 +598,12 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { char hashstr[40] = {0}; SRpcHead *pHead = (SRpcHead *)pRecv->msg; - size_t size = sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); + /*size_t size = */sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); // check if it is already allocated - SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); + SRpcConn **ppConn = (SRpcConn **)(taosGetStrHashData(pRpc->hash, hashstr)); + +// SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); if (ppConn) pConn = *ppConn; if (pConn) return pConn; @@ -627,8 +634,10 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { pRpc->index = (pRpc->index+1) % pRpc->numOfThreads; pConn->localPort = (pRpc->localPort + pRpc->index); } - - taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); + + taosAddStrHash(pRpc->hash, hashstr, (char *)&pConn); + +// taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); tTrace("%s %p, rpc connection is allocated, sid:%d id:%s port:%u", pRpc->label, pConn, sid, pConn->user, pConn->localPort); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index cf86ddbcd2..1e0d8e1609 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -560,7 +560,6 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock); -// pQueryHandle->rhelper.pDataCols[0]->cols[0]; SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; int32_t endPos = cur->pos; @@ -607,8 +606,10 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j); if (pCol->info.colId == colId) { - memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start, - pQueryHandle->realNumOfRows * pCol->info.bytes); + memmove(pCol->pData, &pQueryHandle->rhelper.pDataCols[0]->cols[i], + sizeof(SDataCol) + pQueryHandle->rhelper.pDataCols[0]->cols[i].len); + + tdPopDataColsPoints(pCol->pData, start); break; } } diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 71ce472bd8..63fe124de2 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -79,7 +79,7 @@ static FORCE_INLINE void taosFreeNode(void *data) { */ static SCacheDataNode *taosCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { - size_t totalSize = size + sizeof(SCacheDataNode) + keyLen; + size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1; SCacheDataNode *pNewNode = calloc(1, totalSize); if (pNewNode == NULL) { diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 24098aabc6..9f66eba37a 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -10,7 +10,6 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) -# ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) - ADD_EXECUTABLE(utilTest hashTest.cpp cacheTest.cpp) + ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread) ENDIF() \ No newline at end of file diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 6bade385bd..b955401e43 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -75,9 +75,9 @@ int main(int argc, char *argv[]) { doQuery(taos, "create database if not exists test"); doQuery(taos, "use test"); -// doQuery(taos, "select a from m1"); -// doQuery(taos, "select m2.u from m1, m2 where m1.ts=m2.ts and m1.a=m2.b;"); - doQuery(taos, "select last_row(ts) from tm99"); +// doQuery(taos, "create table t1(ts timestamp, k binary(12), f nchar(2))"); + doQuery(taos, "select m1.ts,m1.a from m1, m2 where m1.ts=m2.ts and m1.a=m2.b;"); +// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 'abc')"); // doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);"); -- GitLab