From 8513dc10672a245578742680ceb1dd596da3fae8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 4 Nov 2022 15:21:38 +0800 Subject: [PATCH] enh: insert optimize --- include/libs/catalog/catalog.h | 7 +- include/libs/nodes/querynodes.h | 73 +- include/libs/parser/parser.h | 4 +- source/client/inc/clientInt.h | 3 +- source/client/src/clientImpl.c | 16 +- source/client/src/clientMain.c | 111 +- source/libs/nodes/src/nodesUtilFuncs.c | 19 +- source/libs/parser/inc/parInsertUtil.h | 27 +- source/libs/parser/inc/parInt.h | 3 +- source/libs/parser/inc/parUtil.h | 38 +- source/libs/parser/src/parAstParser.c | 1 + source/libs/parser/src/parInsertSml.c | 6 +- source/libs/parser/src/parInsertSql.c | 2329 +++++++++-------- source/libs/parser/src/parInsertStmt.c | 28 +- source/libs/parser/src/parInsertUtil.c | 86 +- source/libs/parser/src/parUtil.c | 177 +- source/libs/parser/src/parser.c | 36 +- source/libs/parser/test/mockCatalog.cpp | 15 + .../libs/parser/test/mockCatalogService.cpp | 26 +- source/libs/parser/test/mockCatalogService.h | 4 +- source/libs/parser/test/parTestUtil.cpp | 201 +- source/libs/scheduler/src/schRemote.c | 2 +- 22 files changed, 1640 insertions(+), 1572 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 9f1513d100..7ce5a58a00 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -186,6 +186,8 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, */ int32_t catalogGetTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + STableMeta** pTableMeta); /** * Get a super table's meta data. @@ -198,6 +200,8 @@ int32_t catalogGetTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const S */ int32_t catalogGetSTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + STableMeta** pTableMeta); int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); @@ -261,7 +265,8 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, SRequestConnInfo* pConn, c * @return error code */ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pName, SVgroupInfo* vgInfo); - +int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + SVgroupInfo* pVgroup, bool* exists); /** * Get all meta data required in pReq. * @param pCatalog (input, got with catalogGetHandle) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 837834f795..a85f8bf63d 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -354,12 +354,33 @@ typedef struct SVgDataBlocks { void* pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; +typedef void (*FFreeDataBlockHash)(SHashObj*); +typedef void (*FFreeDataBlockArray)(SArray*); + typedef struct SVnodeModifOpStmt { - ENodeType nodeType; - ENodeType sqlNodeType; - SArray* pDataBlocks; // data block for each vgroup, SArray. - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* sql; // current sql statement position + ENodeType nodeType; + ENodeType sqlNodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* pSql; // current sql statement position + int32_t totalRowsNum; + int32_t totalTbNum; + SName targetTableName; + SName usingTableName; + const char* pBoundCols; + struct STableMeta* pTableMeta; + SHashObj* pVgroupsHashObj; + SHashObj* pTableBlockHashObj; + SHashObj* pSubTableHashObj; + SHashObj* pTableNameHashObj; + SHashObj* pDbFNameHashObj; + SArray* pVgDataBlocks; + SVCreateTbReq createTblReq; + TdFilePtr fp; + FFreeDataBlockHash freeHashFunc; + FFreeDataBlockArray freeArrayFunc; + bool usingTableProcessing; + bool fileProcessing; } SVnodeModifOpStmt; typedef struct SExplainOptions { @@ -389,24 +410,32 @@ typedef enum EQueryExecMode { QUERY_EXEC_MODE_EMPTY_RESULT } EQueryExecMode; +typedef enum EQueryExecStage { + QUERY_EXEC_STAGE_PARSE = 1, + QUERY_EXEC_STAGE_ANALYSE, + QUERY_EXEC_STAGE_SCHEDULE, + QUERY_EXEC_STAGE_END +} EQueryExecStage; + typedef struct SQuery { - ENodeType type; - EQueryExecMode execMode; - bool haveResultSet; - SNode* pRoot; - int32_t numOfResCols; - SSchema* pResSchema; - int8_t precision; - SCmdMsgInfo* pCmdMsg; - int32_t msgType; - SArray* pTargetTableList; - SArray* pTableList; - SArray* pDbList; - bool showRewrite; - int32_t placeholderNum; - SArray* pPlaceholderValues; - SNode* pPrepareRoot; - bool stableQuery; + ENodeType type; + EQueryExecStage execStage; + EQueryExecMode execMode; + bool haveResultSet; + SNode* pRoot; + int32_t numOfResCols; + SSchema* pResSchema; + int8_t precision; + SCmdMsgInfo* pCmdMsg; + int32_t msgType; + SArray* pTargetTableList; + SArray* pTableList; + SArray* pDbList; + bool showRewrite; + int32_t placeholderNum; + SArray* pPlaceholderValues; + SNode* pPrepareRoot; + bool stableQuery; } SQuery; void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index bcd2316baf..1a7e6dc748 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -64,8 +64,6 @@ typedef struct SParseContext { SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos int64_t allocatorId; - bool needMultiParse; - SParseCsvCxt csvCxt; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); @@ -75,6 +73,8 @@ bool qIsInsertValuesSql(const char* pStr, size_t length); int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq); int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, SQuery* pQuery); +int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, + SQuery* pQuery); void qDestroyParseContext(SParseContext* pCxt); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cf8b0babdd..3d99fc435a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -379,7 +379,6 @@ void hbMgrInitMqHbRspHandle(); typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; SCatalogReq* pCatalogReq; - SMetaData* pResultMeta; SRequestObj* pRequest; } SSqlCallbackWrapper; @@ -393,7 +392,7 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList); int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog); int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog); bool qnodeRequired(SRequestObj* pRequest); -int32_t continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest); +void continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest); void destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper); #ifdef __cplusplus diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c12d78c18..703ca7d951 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -867,6 +867,10 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { return code; } +static bool incompletaFileParsing(SNode* pStmt) { + return QUERY_NODE_VNODE_MODIF_STMT != nodeType(pStmt) ? false : ((SVnodeModifOpStmt*)pStmt)->fileProcessing; +} + // todo refacto the error code mgmt void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SSqlCallbackWrapper* pWrapper = param; @@ -921,11 +925,9 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { pRequest->code = code1; } - if (pRequest->code == TSDB_CODE_SUCCESS && NULL != pWrapper->pParseCtx && pWrapper->pParseCtx->needMultiParse) { - code = continueInsertFromCsv(pWrapper, pRequest); - if (TSDB_CODE_SUCCESS == code) { - return; - } + if (pRequest->code == TSDB_CODE_SUCCESS && incompletaFileParsing(pRequest->pQuery->pRoot)) { + continueInsertFromCsv(pWrapper, pRequest); + return; } destorySqlCallbackWrapper(pWrapper); @@ -1049,7 +1051,9 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat } if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; - buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + if (QUERY_NODE_VNODE_MODIF_STMT != nodeType(pRequest->pQuery->pRoot)) { + buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + } SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter, .requestId = pRequest->requestId, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b03576ff01..2b2ee03638 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -682,11 +682,10 @@ void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) { } destoryCatalogReq(pWrapper->pCatalogReq); qDestroyParseContext(pWrapper->pParseCtx); - catalogFreeMetaData(pWrapper->pResultMeta); taosMemoryFree(pWrapper); } -void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { +static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t code) { SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param; SRequestObj *pRequest = pWrapper->pRequest; SQuery *pQuery = pRequest->pQuery; @@ -704,13 +703,6 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->metric.semanticEnd = taosGetTimestampUs(); - if (code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) { - pWrapper->pResultMeta = catalogCloneMetaData(pResultMeta); - if (NULL == pWrapper->pResultMeta) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } - if (code == TSDB_CODE_SUCCESS) { if (pQuery->haveResultSet) { setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols); @@ -747,14 +739,83 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { } } -int32_t continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) { - qDestroyQuery(pRequest->pQuery); - pRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pRequest->pQuery) { - return TSDB_CODE_OUT_OF_MEMORY; +static int32_t getAllMetaAsync(SSqlCallbackWrapper *pWrapper, catalogCallback fp) { + SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter, + .requestId = pWrapper->pParseCtx->requestId, + .requestObjRefId = pWrapper->pParseCtx->requestRid, + .mgmtEps = pWrapper->pParseCtx->mgmtEpSet}; + + pWrapper->pRequest->metric.ctgStart = taosGetTimestampUs(); + + return catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, fp, pWrapper, + &pWrapper->pRequest->body.queryJob); +} + +static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t code); + +static int32_t phaseAsyncQuery(SSqlCallbackWrapper *pWrapper) { + int32_t code = TSDB_CODE_SUCCESS; + switch (pWrapper->pRequest->pQuery->execStage) { + case QUERY_EXEC_STAGE_PARSE: { + // continue parse after get metadata + code = getAllMetaAsync(pWrapper, doAsyncQueryFromParse); + break; + } + case QUERY_EXEC_STAGE_ANALYSE: { + // analysis after get metadata + code = getAllMetaAsync(pWrapper, doAsyncQueryFromAnalyse); + break; + } + case QUERY_EXEC_STAGE_SCHEDULE: { + launchAsyncQuery(pWrapper->pRequest, pWrapper->pRequest->pQuery, NULL, pWrapper); + break; + } + default: + break; + } + return code; +} + +static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t code) { + SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param; + SRequestObj *pRequest = pWrapper->pRequest; + SQuery *pQuery = pRequest->pQuery; + + pRequest->metric.ctgEnd = taosGetTimestampUs(); + qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); + + if (code == TSDB_CODE_SUCCESS) { + code = qContinueParseSql(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); + } + + if (TSDB_CODE_SUCCESS == code) { + code = phaseAsyncQuery(pWrapper); + } + + if (TSDB_CODE_SUCCESS != code) { + tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, + tstrerror(code), pWrapper->pRequest->requestId); + destorySqlCallbackWrapper(pWrapper); + terrno = code; + pWrapper->pRequest->code = code; + pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code); + } +} + +void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) { + int32_t code = qParseSqlSyntax(pWrapper->pParseCtx, &pRequest->pQuery, pWrapper->pCatalogReq); + if (TSDB_CODE_SUCCESS == code) { + code = phaseAsyncQuery(pWrapper); + } + + if (TSDB_CODE_SUCCESS != code) { + tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, + tstrerror(code), pWrapper->pRequest->requestId); + destorySqlCallbackWrapper(pWrapper); + terrno = code; + pWrapper->pRequest->code = code; + pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code); } - retrieveMetaCallback(pWrapper->pResultMeta, pWrapper, TSDB_CODE_SUCCESS); - return TSDB_CODE_SUCCESS; } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { @@ -836,26 +897,16 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (TSDB_CODE_SUCCESS == code && !updateMetaForce) { SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; - if (NULL == pRequest->pQuery->pRoot) { + if (QUERY_NODE_INSERT_STMT == nodeType(pRequest->pQuery->pRoot)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - } else if (QUERY_NODE_SELECT_STMT == pRequest->pQuery->pRoot->type) { + } else if (QUERY_NODE_SELECT_STMT == nodeType(pRequest->pQuery->pRoot)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfQueryReq, 1); } } if (TSDB_CODE_SUCCESS == code) { - SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter, - .requestId = pWrapper->pParseCtx->requestId, - .requestObjRefId = pWrapper->pParseCtx->requestRid, - .mgmtEps = pWrapper->pParseCtx->mgmtEpSet}; - - pRequest->metric.ctgStart = taosGetTimestampUs(); - - code = catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, retrieveMetaCallback, - pWrapper, &pRequest->body.queryJob); - } - - if (TSDB_CODE_SUCCESS != code) { + phaseAsyncQuery(pWrapper); + } else { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 0c13dd822b..778bfcdf82 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -791,9 +791,24 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode((SNode*)pStmt->pSlimit); break; } - case QUERY_NODE_VNODE_MODIF_STMT: - destroyVgDataBlockArray(((SVnodeModifOpStmt*)pNode)->pDataBlocks); + case QUERY_NODE_VNODE_MODIF_STMT: { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pNode; + destroyVgDataBlockArray(pStmt->pDataBlocks); + taosMemoryFreeClear(pStmt->pTableMeta); + taosHashCleanup(pStmt->pVgroupsHashObj); + taosHashCleanup(pStmt->pSubTableHashObj); + taosHashCleanup(pStmt->pTableNameHashObj); + taosHashCleanup(pStmt->pDbFNameHashObj); + if (pStmt->freeHashFunc) { + pStmt->freeHashFunc(pStmt->pTableBlockHashObj); + } + if (pStmt->freeArrayFunc) { + pStmt->freeArrayFunc(pStmt->pVgDataBlocks); + } + tdDestroySVCreateTbReq(&pStmt->createTblReq); + taosCloseFile(&pStmt->fp); break; + } case QUERY_NODE_CREATE_DATABASE_STMT: nodesDestroyNode((SNode*)((SCreateDatabaseStmt*)pNode)->pOptions); break; diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 1e941632e7..f747fdc2c9 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -79,29 +79,6 @@ typedef struct SInsertParseBaseContext { SMsgBuf msg; } SInsertParseBaseContext; -typedef struct SInsertParseContext { - SParseContext *pComCxt; // input - char *pSql; // input - SMsgBuf msg; // input - STableMeta *pTableMeta; // each table - SParsedDataColInfo tags; // each table - SVCreateTbReq createTblReq; // each table - SHashObj *pVgroupsHashObj; // global - SHashObj *pTableBlockHashObj; // global - SHashObj *pSubTableHashObj; // global - SArray *pVgDataBlocks; // global - SHashObj *pTableNameHashObj; // global - SHashObj *pDbFNameHashObj; // global - int32_t totalNum; - SVnodeModifOpStmt *pOutput; - SStmtCallback *pStmtCb; - SParseMetaCache *pMetaCache; - char sTableName[TSDB_TABLE_NAME_LEN]; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; - int64_t memElapsed; - int64_t parRowElapsed; -} SInsertParseContext; - typedef struct SInsertParseSyntaxCxt { SParseContext *pComCxt; char *pSql; @@ -142,7 +119,7 @@ typedef struct STableDataBlocks { int32_t insGetExtendedRowSize(STableDataBlocks *pBlock); void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx); -int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows); +int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows, SMsgBuf *pMsg); int32_t insSchemaIdxCompar(const void *lhs, const void *rhs); int32_t insBoundIdxCompar(const void *lhs, const void *rhs); void insSetBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); @@ -161,7 +138,7 @@ void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag SArray *tagName, uint8_t tagNum); int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); -int32_t insBuildOutput(SInsertParseContext *pCxt); +int32_t insBuildOutput(SVnodeModifOpStmt *pStmt); void insDestroyDataBlock(STableDataBlocks *pDataBlock); #endif // TDENGINE_PAR_INSERT_UTIL_H diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index b799fa9855..66aec272d7 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -27,8 +27,7 @@ extern "C" { #define QUERY_SMA_OPTIMIZE_DISABLE 0 #define QUERY_SMA_OPTIMIZE_ENABLE 1 -int32_t parseInsertSyntax(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache); -int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache); +int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData); int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 75b631ec9d..10a86866d5 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -60,22 +60,17 @@ typedef struct SInsertTablesMetaReq { } SInsertTablesMetaReq; typedef struct SParseMetaCache { - SHashObj* pTableMeta; // key is tbFName, element is STableMeta* - SHashObj* pDbVgroup; // key is dbFName, element is SArray* - SHashObj* pTableVgroup; // key is tbFName, element is SVgroupInfo* - SHashObj* pDbCfg; // key is tbFName, element is SDbCfgInfo* - SHashObj* pDbInfo; // key is tbFName, element is SDbInfo* - SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass - SHashObj* pUdf; // key is funcName, element is SFuncInfo* - SHashObj* pTableIndex; // key is tbFName, element is SArray* - SHashObj* pTableCfg; // key is tbFName, element is STableCfg* - SArray* pDnodes; // element is SEpSet - bool dnodeRequired; - SHashObj* pInsertTables; // key is dbName, element is SInsertTablesMetaReq*, for insert - const char* pUser; - const SArray* pTableMetaData; // pRes = STableMeta* - const SArray* pTableVgroupData; // pRes = SVgroupInfo* - int32_t sqlTableNum; + SHashObj* pTableMeta; // key is tbFName, element is STableMeta* + SHashObj* pDbVgroup; // key is dbFName, element is SArray* + SHashObj* pTableVgroup; // key is tbFName, element is SVgroupInfo* + SHashObj* pDbCfg; // key is tbFName, element is SDbCfgInfo* + SHashObj* pDbInfo; // key is tbFName, element is SDbInfo* + SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass + SHashObj* pUdf; // key is funcName, element is SFuncInfo* + SHashObj* pTableIndex; // key is tbFName, element is SArray* + SHashObj* pTableCfg; // key is tbFName, element is STableCfg* + SArray* pDnodes; // element is SEpSet + bool dnodeRequired; } SParseMetaCache; int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); @@ -93,9 +88,8 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); -int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); -int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, - bool insertValuesStmt); +int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); +int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache); int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache); int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache); @@ -122,12 +116,6 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); -int32_t reserveTableMetaInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo, - SParseMetaCache* pMetaCache); -int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* pMetaCache, int32_t tableNo, - STableMeta** pMeta); -int32_t getTableVgroupFromCacheForInsert(SArray* pTableVgroupPos, SParseMetaCache* pMetaCache, int32_t tableNo, - SVgroupInfo* pVgroup); void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request); #ifdef __cplusplus diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 28dc6179f9..bb3d1a2cb3 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -84,6 +84,7 @@ abort_parse: (*pQuery)->pRoot = cxt.pRootNode; (*pQuery)->placeholderNum = cxt.placeholderNo; TSWAP((*pQuery)->pPlaceholderValues, cxt.pPlaceholderValues); + (*pQuery)->execStage = QUERY_EXEC_STAGE_ANALYSE; } taosArrayDestroy(cxt.pPlaceholderValues); return cxt.errCode; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 8be5b86b37..d18b11ad57 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -333,11 +333,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, rowNum)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX"); - } - - return TSDB_CODE_SUCCESS; + return insSetBlockInfo(pBlocks, pDataBlock, rowNum, &pBuf); } void* smlInitHandle(SQuery* pQuery) { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 1ccb34cd1f..d9bfee24d1 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -18,123 +18,246 @@ #include "tglobal.h" #include "ttime.h" -#define NEXT_TOKEN_WITH_PREV(pSql, sToken) \ - do { \ - int32_t index = 0; \ - sToken = tStrGetToken(pSql, &index, true); \ - pSql += index; \ +#define NEXT_TOKEN_WITH_PREV(pSql, token) \ + do { \ + int32_t index = 0; \ + token = tStrGetToken(pSql, &index, true); \ + pSql += index; \ } while (0) -#define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \ - do { \ - sToken = tStrGetToken(pSql, &index, false); \ +#define NEXT_TOKEN_KEEP_SQL(pSql, token, index) \ + do { \ + token = tStrGetToken(pSql, &index, false); \ } while (0) -#define NEXT_VALID_TOKEN(pSql, sToken) \ - do { \ - sToken.n = tGetToken(pSql, &sToken.type); \ - sToken.z = pSql; \ - pSql += sToken.n; \ - } while (TK_NK_SPACE == sToken.type) +#define NEXT_VALID_TOKEN(pSql, token) \ + do { \ + (token).n = tGetToken(pSql, &(token).type); \ + (token).z = (char*)pSql; \ + pSql += (token).n; \ + } while (TK_NK_SPACE == (token).type) + +typedef struct SInsertParseContext { + SParseContext* pComCxt; + SMsgBuf msg; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; + SParsedDataColInfo tags; // for stmt + bool missCache; +} SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; -static int32_t skipInsertInto(char** pSql, SMsgBuf* pMsg) { - SToken sToken; - NEXT_TOKEN(*pSql, sToken); - if (TK_INSERT != sToken.type && TK_IMPORT != sToken.type) { - return buildSyntaxErrMsg(pMsg, "keyword INSERT is expected", sToken.z); +static bool isNullStr(SToken* pToken) { + return ((pToken->type == TK_NK_STRING) && (strlen(TSDB_DATA_NULL_STR_L) == pToken->n) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); +} + +static bool isNullValue(int8_t dataType, SToken* pToken) { + return TK_NULL == pToken->type || (!IS_STR_DATA_TYPE(dataType) && isNullStr(pToken)); +} + +static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { + errno = 0; + *value = taosStr2Double(pToken->z, endPtr); + + // not a valid integer number, return error + if ((*endPtr - pToken->z) != pToken->n) { + return TK_NK_ILLEGAL; + } + + return pToken->type; +} + +static int32_t skipInsertInto(const char** pSql, SMsgBuf* pMsg) { + SToken token; + NEXT_TOKEN(*pSql, token); + if (TK_INSERT != token.type && TK_IMPORT != token.type) { + return buildSyntaxErrMsg(pMsg, "keyword INSERT is expected", token.z); } - NEXT_TOKEN(*pSql, sToken); - if (TK_INTO != sToken.type) { - return buildSyntaxErrMsg(pMsg, "keyword INTO is expected", sToken.z); + NEXT_TOKEN(*pSql, token); + if (TK_INTO != token.type) { + return buildSyntaxErrMsg(pMsg, "keyword INTO is expected", token.z); } return TSDB_CODE_SUCCESS; } -static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass) { - SParseContext* pBasicCtx = pCxt->pComCxt; - if (pBasicCtx->async) { - return getUserAuthFromCache(pCxt->pMetaCache, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass); +static int32_t skipParentheses(SInsertParseContext* pCxt, const char** pSql) { + SToken token; + int32_t expectRightParenthesis = 1; + while (1) { + NEXT_TOKEN(*pSql, token); + if (TK_NK_LP == token.type) { + ++expectRightParenthesis; + } else if (TK_NK_RP == token.type && 0 == --expectRightParenthesis) { + break; + } + if (0 == token.n) { + return buildSyntaxErrMsg(&pCxt->msg, ") expected", NULL); + } } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, - .requestId = pBasicCtx->requestId, - .requestObjRefId = pBasicCtx->requestRid, - .mgmtEps = pBasicCtx->mgmtEpSet}; + return TSDB_CODE_SUCCESS; +} - return catalogChkAuth(pBasicCtx->pCatalog, &conn, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass); +static int32_t skipTableOptions(SInsertParseContext* pCxt, const char** pSql) { + do { + int32_t index = 0; + SToken token; + NEXT_TOKEN_KEEP_SQL(*pSql, token, index); + if (TK_TTL == token.type || TK_COMMENT == token.type) { + *pSql += index; + NEXT_TOKEN_WITH_PREV(*pSql, token); + } else { + break; + } + } while (1); + return TSDB_CODE_SUCCESS; } -static int32_t getTableSchema(SInsertParseContext* pCxt, int32_t tbNo, SName* pTbName, bool isStb, - STableMeta** pTableMeta) { - SParseContext* pBasicCtx = pCxt->pComCxt; - if (pBasicCtx->async) { - return getTableMetaFromCacheForInsert(pBasicCtx->pTableMetaPos, pCxt->pMetaCache, tbNo, pTableMeta); +// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) +static int32_t ignoreUsingClause(SInsertParseContext* pCxt, const char** pSql) { + int32_t code = TSDB_CODE_SUCCESS; + SToken token; + NEXT_TOKEN(*pSql, token); + + NEXT_TOKEN(*pSql, token); + if (TK_NK_LP == token.type) { + code = skipParentheses(pCxt, pSql); + if (TSDB_CODE_SUCCESS == code) { + NEXT_TOKEN(*pSql, token); + } } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, - .requestId = pBasicCtx->requestId, - .requestObjRefId = pBasicCtx->requestRid, - .mgmtEps = pBasicCtx->mgmtEpSet}; - if (isStb) { - return catalogGetSTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta); + // pSql -> TAGS (tag1_value, ...) + if (TSDB_CODE_SUCCESS == code) { + if (TK_TAGS != token.type) { + code = buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", token.z); + } else { + NEXT_TOKEN(*pSql, token); + } + } + if (TSDB_CODE_SUCCESS == code) { + if (TK_NK_LP != token.type) { + code = buildSyntaxErrMsg(&pCxt->msg, "( is expected", token.z); + } else { + code = skipParentheses(pCxt, pSql); + } } - return catalogGetTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta); -} -static int32_t getTableVgroup(SInsertParseContext* pCxt, int32_t tbNo, SName* pTbName, SVgroupInfo* pVg) { - SParseContext* pBasicCtx = pCxt->pComCxt; - if (pBasicCtx->async) { - return getTableVgroupFromCacheForInsert(pBasicCtx->pTableVgroupPos, pCxt->pMetaCache, tbNo, pVg); + if (TSDB_CODE_SUCCESS == code) { + code = skipTableOptions(pCxt, pSql); } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, - .requestId = pBasicCtx->requestId, - .requestObjRefId = pBasicCtx->requestRid, - .mgmtEps = pBasicCtx->mgmtEpSet}; - return catalogGetTableHashVgroup(pBasicCtx->pCatalog, &conn, pTbName, pVg); + + return code; } -static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, bool isStb) { - CHECK_CODE(getTableSchema(pCxt, tbNo, name, isStb, &pCxt->pTableMeta)); - if (!isStb) { - SVgroupInfo vg; - CHECK_CODE(getTableVgroup(pCxt, tbNo, name, &vg)); - CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); +static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pDuplicate) { + *pDuplicate = false; + + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + STableMeta** pMeta = taosHashGet(pStmt->pSubTableHashObj, tbFName, strlen(tbFName)); + if (NULL != pMeta) { + *pDuplicate = true; + int32_t code = ignoreUsingClause(pCxt, &pStmt->pSql); + if (TSDB_CODE_SUCCESS == code) { + return cloneTableMeta(*pMeta, &pStmt->pTableMeta); + } } + return TSDB_CODE_SUCCESS; } -static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) { - return getTableMetaImpl(pCxt, tbNo, name, false); -} +// pStmt->pSql -> field1_name, ...) +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, SParsedDataColInfo* pColList, + SSchema* pSchema) { + col_id_t nCols = pColList->numOfCols; -static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) { - return getTableMetaImpl(pCxt, tbNo, name, true); -} + pColList->numOfBound = 0; + pColList->boundNullLen = 0; + memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); + for (col_id_t i = 0; i < nCols; ++i) { + pColList->cols[i].valStat = VAL_STAT_NONE; + } -static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgInfo* pInfo) { - SParseContext* pBasicCtx = pCxt->pComCxt; - if (pBasicCtx->async) { - CHECK_CODE(getDbCfgFromCache(pCxt->pMetaCache, pDbFName, pInfo)); - } else { - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, - .requestId = pBasicCtx->requestId, - .requestObjRefId = pBasicCtx->requestRid, - .mgmtEps = pBasicCtx->mgmtEpSet}; - CHECK_CODE(catalogGetDBCfg(pBasicCtx->pCatalog, &conn, pDbFName, pInfo)); + SToken token; + bool isOrdered = true; + col_id_t lastColIdx = -1; // last column found + while (1) { + NEXT_TOKEN(*pSql, token); + + if (TK_NK_RP == token.type) { + break; + } + + char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`) + strncpy(tmpTokenBuf, token.z, token.n); + token.z = tmpTokenBuf; + token.n = strdequote(token.z); + + col_id_t t = lastColIdx + 1; + col_id_t index = insFindCol(&token, t, nCols, pSchema); + if (index < 0 && t > 0) { + index = insFindCol(&token, 0, t, pSchema); + isOrdered = false; + } + if (index < 0) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); + } + if (pColList->cols[index].valStat == VAL_STAT_HAS) { + return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); + } + lastColIdx = index; + pColList->cols[index].valStat = VAL_STAT_HAS; + pColList->boundColumns[pColList->numOfBound] = index; + ++pColList->numOfBound; + switch (pSchema[t].type) { + case TSDB_DATA_TYPE_BINARY: + pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); + break; + case TSDB_DATA_TYPE_NCHAR: + pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + break; + default: + pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; + break; + } + } + + pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (!isOrdered) { + pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + if (NULL == pColList->colIdxInfo) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SBoundIdxInfo* pColIdx = pColList->colIdxInfo; + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = pColList->boundColumns[i]; + pColIdx[i].boundIdx = i; + } + taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); + } + + if (pColList->numOfCols > pColList->numOfBound) { + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); } + return TSDB_CODE_SUCCESS; } -static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { - int32_t index = 0; - SToken sToken; - int64_t interval; - int64_t ts = 0; - char* pTokenEnd = *end; +static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { + int32_t index = 0; + int64_t interval; + int64_t ts = 0; + const char* pTokenEnd = *end; if (pToken->type == TK_NOW) { ts = taosGetTimestamp(timePrec); @@ -172,18 +295,17 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time * time expression: * e.g., now+12a, now-5h */ - SToken valueToken; index = 0; - sToken = tStrGetToken(pTokenEnd, &index, false); + SToken token = tStrGetToken(pTokenEnd, &index, false); pTokenEnd += index; - if (sToken.type == TK_NK_MINUS || sToken.type == TK_NK_PLUS) { + if (token.type == TK_NK_MINUS || token.type == TK_NK_PLUS) { index = 0; - valueToken = tStrGetToken(pTokenEnd, &index, false); + SToken valueToken = tStrGetToken(pTokenEnd, &index, false); pTokenEnd += index; if (valueToken.n < 2) { - return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", sToken.z); + return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", token.z); } char unit = 0; @@ -191,7 +313,7 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time return TSDB_CODE_TSC_INVALID_OPERATION; } - if (sToken.type == TK_NK_PLUS) { + if (token.type == TK_NK_PLUS) { ts += interval; } else { ts = ts - interval; @@ -204,92 +326,42 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { - if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && - pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && - pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && - pToken->type != TK_NK_BIN) || - (pToken->n == 0) || (pToken->type == TK_NK_RP)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); - } - - // Remove quotation marks - if (TK_NK_STRING == pToken->type) { - if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { - return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); - } - - int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); - pToken->z = tmpTokenBuf; - pToken->n = len; - } - - return TSDB_CODE_SUCCESS; -} - -static bool isNullStr(SToken* pToken) { - return ((pToken->type == TK_NK_STRING) && (strlen(TSDB_DATA_NULL_STR_L) == pToken->n) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); -} - -static bool isNullValue(int8_t dataType, SToken* pToken) { - return TK_NULL == pToken->type || (!IS_STR_DATA_TYPE(dataType) && isNullStr(pToken)); -} - -static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { - errno = 0; - *value = taosStr2Double(pToken->z, endPtr); - - // not a valid integer number, return error - if ((*endPtr - pToken->z) != pToken->n) { - return TK_NK_ILLEGAL; - } - - return pToken->type; -} - -static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, - _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) { +static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, + SMsgBuf* pMsgBuf) { int64_t iv; uint64_t uv; char* endptr = NULL; - int32_t code = checkAndTrimValue(pToken, tmpTokenBuf, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - if (isNullValue(pSchema->type, pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); } - return func(pMsgBuf, NULL, 0, param); + return TSDB_CODE_SUCCESS; } - if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { - return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z); - } + // strcpy(val->colName, pSchema->name); + val->cid = pSchema->colId; + val->type = pSchema->type; switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - return func(pMsgBuf, &TRUE_VALUE, pSchema->bytes, param); + *(int8_t*)(&val->i64) = TRUE_VALUE; } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - return func(pMsgBuf, &FALSE_VALUE, pSchema->bytes, param); + *(int8_t*)(&val->i64) = FALSE_VALUE; } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, - param); + *(int8_t*)(&val->i64) = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); } else if (pToken->type == TK_NK_FLOAT) { - return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, - param); + *(int8_t*)(&val->i64) = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } + break; } case TSDB_DATA_TYPE_TINYINT: { @@ -299,8 +371,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); } - uint8_t tmpVal = (uint8_t)iv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + *(int8_t*)(&val->i64) = iv; + break; } case TSDB_DATA_TYPE_UTINYINT: { @@ -309,8 +381,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } else if (uv > UINT8_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); } - uint8_t tmpVal = (uint8_t)uv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + *(uint8_t*)(&val->i64) = uv; + break; } case TSDB_DATA_TYPE_SMALLINT: { @@ -319,8 +391,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } else if (!IS_VALID_SMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); } - int16_t tmpVal = (int16_t)iv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + *(int16_t*)(&val->i64) = iv; + break; } case TSDB_DATA_TYPE_USMALLINT: { @@ -329,8 +401,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } else if (uv > UINT16_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); } - uint16_t tmpVal = (uint16_t)uv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + *(uint16_t*)(&val->i64) = uv; + break; } case TSDB_DATA_TYPE_INT: { @@ -339,8 +411,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } else if (!IS_VALID_INT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); } - int32_t tmpVal = (int32_t)iv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + *(int32_t*)(&val->i64) = iv; + break; } case TSDB_DATA_TYPE_UINT: { @@ -349,22 +421,24 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } else if (uv > UINT32_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); } - uint32_t tmpVal = (uint32_t)uv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + *(uint32_t*)(&val->i64) = uv; + break; } case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } - return func(pMsgBuf, &iv, pSchema->bytes, param); + val->i64 = iv; + break; } case TSDB_DATA_TYPE_UBIGINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } - return func(pMsgBuf, &uv, pSchema->bytes, param); + *(uint64_t*)(&val->i64) = uv; + break; } case TSDB_DATA_TYPE_FLOAT: { @@ -376,8 +450,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } - float tmpVal = (float)dv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + *(float*)(&val->i64) = dv; + break; } case TSDB_DATA_TYPE_DOUBLE: { @@ -388,7 +462,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); } - return func(pMsgBuf, &dv, pSchema->bytes, param); + + *(double*)(&val->i64) = dv; + break; } case TSDB_DATA_TYPE_BINARY: { @@ -396,583 +472,750 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - - return func(pMsgBuf, pToken->z, pToken->n, param); + val->pData = strdup(pToken->z); + val->nData = pToken->n; + break; } case TSDB_DATA_TYPE_NCHAR: { - return func(pMsgBuf, pToken->z, pToken->n, param); - } - case TSDB_DATA_TYPE_JSON: { - if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); + int32_t output = 0; + void* p = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z); } - return func(pMsgBuf, pToken->z, pToken->n, param); + val->pData = p; + val->nData = output; + break; } case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t tmpVal; - if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { + if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); } - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + val->i64 = iv; + break; } } - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } -// pSql -> tag1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { - col_id_t nCols = pColList->numOfCols; +// input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ... +// output pStmt->pSql: TAGS (tag1_value, ...) ... +static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SSchema* pTagsSchema = getTableTagSchema(pStmt->pTableMeta); + insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pStmt->pTableMeta)); - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; + SToken token; + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_NK_LP != token.type) { + return TSDB_CODE_SUCCESS; } - SToken sToken; - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found - while (1) { - NEXT_TOKEN(pCxt->pSql, sToken); + pStmt->pSql += index; + return parseBoundColumns(pCxt, &pStmt->pSql, &pCxt->tags, pTagsSchema); +} - if (TK_NK_RP == sToken.type) { - break; +static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, + SArray* pTagName, SArray* pTagVals, STag** pTag) { + if (!isNullValue(pTagSchema->type, pToken)) { + taosArrayPush(pTagName, pTagSchema->name); + } + + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`) - strncpy(tmpTokenBuf, sToken.z, sToken.n); - sToken.z = tmpTokenBuf; - sToken.n = strdequote(sToken.z); + if (isNullValue(pTagSchema->type, pToken)) { + return tTagNew(pTagVals, 1, true, pTag); + } else { + return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg); + } + } - col_id_t t = lastColIdx + 1; - col_id_t index = insFindCol(&sToken, t, nCols, pSchema); - if (index < 0 && t > 0) { - index = insFindCol(&sToken, 0, t, pSchema); - isOrdered = false; + STagVal val = {0}; + int32_t code = + parseTagToken(&pStmt->pSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(pTagVals, &val); + } + + return code; +} + +static void buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) { + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + char stbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->usingTableName, stbFName); + insBuildCreateTbReq(&pStmt->createTblReq, tbFName, pTag, pStmt->pTableMeta->suid, stbFName, pTagName, + pStmt->pTableMeta->tableInfo.numOfTags); +} + +static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { + if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && + pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && + pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && + pToken->type != TK_NK_BIN) || + (pToken->n == 0) || (pToken->type == TK_NK_RP)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); + } + + // Remove quotation marks + if (TK_NK_STRING == pToken->type) { + if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { + return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } - if (index < 0) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, sToken.z); + + int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); + pToken->z = tmpTokenBuf; + pToken->n = len; + } + + return TSDB_CODE_SUCCESS; +} + +// pSql -> tag1_value, ...) +static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + SSchema* pSchema = getTableTagSchema(pStmt->pTableMeta); + SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); + SArray* pTagName = taosArrayInit(8, TSDB_COL_NAME_LEN); + SToken token; + bool isParseBindParam = false; + bool isJson = false; + STag* pTag = NULL; + for (int i = 0; TSDB_CODE_SUCCESS == code && i < pCxt->tags.numOfBound; ++i) { + NEXT_TOKEN_WITH_PREV(pStmt->pSql, token); + + if (token.type == TK_NK_QUESTION) { + isParseBindParam = true; + if (NULL == pCxt->pComCxt->pStmtCb) { + code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", token.z); + break; + } + + continue; } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { - return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", sToken.z); + + if (isParseBindParam) { + code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); + break; } - lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; + + SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; + isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; + code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + code = parseTagValue(pCxt, pStmt, pTagSchema, &token, pTagName, pTagVals, &pTag); } } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) { + code = tTagNew(pTagVals, 1, false, &pTag); + } - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { + buildCreateTbReq(pStmt, pTag, pTagName); + pTag = NULL; + } + + for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { + STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); + if (IS_VAR_DATA_TYPE(p->type)) { + taosMemoryFreeClear(p->pData); } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; + } + taosArrayDestroy(pTagVals); + taosArrayDestroy(pTagName); + tTagFree(pTag); + return code; +} + +// input pStmt->pSql: TAGS (tag1_value, ...) [table_options] ... +// output pStmt->pSql: [table_options] ... +static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + NEXT_TOKEN(pStmt->pSql, token); + if (TK_TAGS != token.type) { + return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", token.z); + } + + NEXT_TOKEN(pStmt->pSql, token); + if (TK_NK_LP != token.type) { + return buildSyntaxErrMsg(&pCxt->msg, "( is expected", token.z); + } + + int32_t code = parseTagsClauseImpl(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + NEXT_VALID_TOKEN(pStmt->pSql, token); + if (TK_NK_COMMA == token.type) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_TAGS_NOT_MATCHED); + } else if (TK_NK_RP != token.type) { + code = buildSyntaxErrMsg(&pCxt->msg, ") is expected", token.z); } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); } + return code; +} - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); +static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + pStmt->pTableMeta->uid = pStmt->totalTbNum; + pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; + + STableMeta* pBackup = NULL; + if (TSDB_CODE_SUCCESS != cloneTableMeta(pStmt->pTableMeta, &pBackup)) { + return TSDB_CODE_OUT_OF_MEMORY; } + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + return taosHashPut(pStmt->pSubTableHashObj, tbFName, strlen(tbFName), &pBackup, POINTER_BYTES); +} + +static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + do { + int32_t index = 0; + SToken token; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_TTL == token.type) { + pStmt->pSql += index; + NEXT_TOKEN_WITH_PREV(pStmt->pSql, token); + if (TK_NK_INTEGER != token.type) { + return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); + } + pStmt->createTblReq.ttl = taosStr2Int32(token.z, NULL, 10); + if (pStmt->createTblReq.ttl < 0) { + return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); + } + } else if (TK_COMMENT == token.type) { + pStmt->pSql += index; + NEXT_TOKEN(pStmt->pSql, token); + if (TK_NK_STRING != token.type) { + return buildSyntaxErrMsg(&pCxt->msg, "Invalid option comment", token.z); + } + if (token.n >= TSDB_TB_COMMENT_LEN) { + return buildSyntaxErrMsg(&pCxt->msg, "comment too long", token.z); + } + int32_t len = trimString(token.z, token.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); + pStmt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len); + if (NULL == pStmt->createTblReq.comment) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pStmt->createTblReq.commentLen = len; + } else { + break; + } + } while (1); return TSDB_CODE_SUCCESS; } -static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, - SMsgBuf* pMsgBuf) { - int64_t iv; - uint64_t uv; - char* endptr = NULL; +// input pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +// output pStmt->pSql: +// 1. [(field1_name, ...)] +// 2. VALUES ... | FILE ... +static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + if ('\0' == pStmt->usingTableName.tname[0]) { + return TSDB_CODE_SUCCESS; + } - if (isNullValue(pSchema->type, pToken)) { - if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { - return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); + int32_t code = parseBoundTagsClause(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = parseTagsClause(pCxt, pStmt); + } + if (TSDB_CODE_SUCCESS == code) { + code = parseTableOptions(pCxt, pStmt); + } + + return code; +} + +static int32_t checkAuth(SParseContext* pCxt, SName* pTbName) { + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTbName, dbFName); + SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, + .requestId = pCxt->requestId, + .requestObjRefId = pCxt->requestRid, + .mgmtEps = pCxt->mgmtEpSet}; + int32_t code = TSDB_CODE_SUCCESS; + bool pass = true; + if (pCxt->async) { + // todo replace with cached api + code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass); + } else { + code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass); + } + if (TSDB_CODE_SUCCESS == code && !pass) { + code = TSDB_CODE_PAR_PERMISSION_DENIED; + } + return code; +} + +static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta, + bool* pMissCache) { + SParseContext* pComCxt = pCxt->pComCxt; + SRequestConnInfo conn = {.pTrans = pComCxt->pTransporter, + .requestId = pComCxt->requestId, + .requestObjRefId = pComCxt->requestRid, + .mgmtEps = pComCxt->mgmtEpSet}; + int32_t code = TSDB_CODE_SUCCESS; + if (pComCxt->async) { + if (isStb) { + code = catalogGetCachedSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + } else { + code = catalogGetCachedTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + } + } else { + if (isStb) { + code = catalogGetSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + } else { + code = catalogGetTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); } + } + if (TSDB_CODE_SUCCESS == code) { + if (NULL == *pTableMeta) { + *pMissCache = true; + } else if (isStb && TSDB_SUPER_TABLE != (*pTableMeta)->tableType) { + code = buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); + } + } + return code; +} + +static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool isStb, bool* pMissCache) { + SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, + .requestId = pCxt->requestId, + .requestObjRefId = pCxt->requestRid, + .mgmtEps = pCxt->mgmtEpSet}; + int32_t code = TSDB_CODE_SUCCESS; + SVgroupInfo vg; + bool exists = true; + if (pCxt->async) { + code = catalogGetCachedTableHashVgroup(pCxt->pCatalog, &conn, &pStmt->targetTableName, &vg, &exists); + } else { + code = catalogGetTableHashVgroup(pCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); + } + if (TSDB_CODE_SUCCESS == code) { + if (exists) { + if (isStb) { + pStmt->pTableMeta->vgId = vg.vgId; + } + code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); + } + *pMissCache = !exists; + } + return code; +} +static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName); + if (TSDB_CODE_SUCCESS == code) { + code = getTableMeta(pCxt, &pStmt->targetTableName, false, &pStmt->pTableMeta, &pCxt->missCache); + } + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); + } + return code; +} + +static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + return insCreateSName(&pStmt->usingTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); +} + +static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName); + if (TSDB_CODE_SUCCESS == code) { + code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache); + } + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = getTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); + } + return code; +} + +static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + NEXT_TOKEN(pStmt->pSql, token); + int32_t code = preParseUsingTableName(pCxt, pStmt, &token); + if (TSDB_CODE_SUCCESS == code) { + code = getUsingTableSchema(pCxt, pStmt); + } + if (TSDB_CODE_SUCCESS == code) { + code = storeTableMeta(pCxt, pStmt); + } + return code; +} + +// input pStmt->pSql: +// 1(care). [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ... +// 2. VALUES ... | FILE ... +// output pStmt->pSql: +// 1. [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ... +// 2. VALUES ... | FILE ... +static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_USING != token.type) { + return getTargetTableSchema(pCxt, pStmt); + } + + // pStmt->pSql -> stb_name [(tag1_name, ...) + pStmt->pSql += index; + bool duplicate = false; + int32_t code = parseDuplicateUsingClause(pCxt, pStmt, &duplicate); + if (TSDB_CODE_SUCCESS == code && !duplicate) { + return parseUsingTableNameImpl(pCxt, pStmt); + } + return code; +} + +static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + return insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); +} + +// input pStmt->pSql: +// 1(care). [(field1_name, ...)] ... +// 2. [ USING ... ] ... +// 3. VALUES ... | FILE ... +// output pStmt->pSql: +// 1. [ USING ... ] ... +// 2. VALUES ... | FILE ... +static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_NK_LP != token.type) { return TSDB_CODE_SUCCESS; } - // strcpy(val->colName, pSchema->name); - val->cid = pSchema->colId; - val->type = pSchema->type; + // pStmt->pSql -> field1_name, ...) + pStmt->pSql += index; + pStmt->pBoundCols = pStmt->pSql; + return skipParentheses(pCxt, &pStmt->pSql); +} + +static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks** pDataBuf) { + if (pCxt->pComCxt->async) { + return insGetDataBlockFromList(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), + getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL, + &pStmt->createTblReq); + } + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + return insGetDataBlockFromList(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, + pDataBuf, NULL, &pStmt->createTblReq); +} + +static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, + STableDataBlocks* pDataBuf) { + SToken token; + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_NK_LP == token.type) { + pStmt->pSql += index; + if (NULL != pStmt->pBoundCols) { + return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); + } + // pStmt->pSql -> field1_name, ...) + return parseBoundColumns(pCxt, &pStmt->pSql, &pDataBuf->boundColumnInfo, getTableColumnSchema(pStmt->pTableMeta)); + } + + if (NULL != pStmt->pBoundCols) { + return parseBoundColumns(pCxt, &pStmt->pBoundCols, &pDataBuf->boundColumnInfo, + getTableColumnSchema(pStmt->pTableMeta)); + } + + return TSDB_CODE_SUCCESS; +} + +// input pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +// output pStmt->pSql: VALUES ... | FILE ... +static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, + STableDataBlocks** pDataBuf) { + int32_t code = parseUsingClauseBottom(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = getTableDataBlocks(pCxt, pStmt, pDataBuf); + } + if (TSDB_CODE_SUCCESS == code) { + code = parseBoundColumnsClause(pCxt, pStmt, *pDataBuf); + } + return code; +} + +// input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... +// output pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + int32_t code = preParseTargetTableName(pCxt, pStmt, pTbName); + if (TSDB_CODE_SUCCESS == code) { + // option: [(field1_name, ...)] + code = preParseBoundColumnsClause(pCxt, pStmt); + } + if (TSDB_CODE_SUCCESS == code) { + // option: [USING stb_name] + code = parseUsingTableName(pCxt, pStmt); + } + return code; +} + +static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, + int16_t timePrec, _row_append_fn_t func, void* param) { + int64_t iv; + uint64_t uv; + char* endptr = NULL; switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - *(int8_t*)(&val->i64) = TRUE_VALUE; + return func(&pCxt->msg, &TRUE_VALUE, pSchema->bytes, param); } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - *(int8_t*)(&val->i64) = FALSE_VALUE; + return func(&pCxt->msg, &FALSE_VALUE, pSchema->bytes, param); } else { - return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - *(int8_t*)(&val->i64) = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); + return func(&pCxt->msg, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), + pSchema->bytes, param); } else if (pToken->type == TK_NK_FLOAT) { - *(int8_t*)(&val->i64) = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); + return func(&pCxt->msg, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, + param); } else { - return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } - break; } case TSDB_DATA_TYPE_TINYINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); } - *(int8_t*)(&val->i64) = iv; - break; + uint8_t tmpVal = (uint8_t)iv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_UTINYINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); } else if (uv > UINT8_MAX) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); } - *(uint8_t*)(&val->i64) = uv; - break; + uint8_t tmpVal = (uint8_t)uv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_SMALLINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); } - *(int16_t*)(&val->i64) = iv; - break; + int16_t tmpVal = (int16_t)iv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_USMALLINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); } else if (uv > UINT16_MAX) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); } - *(uint16_t*)(&val->i64) = uv; - break; + uint16_t tmpVal = (uint16_t)uv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_INT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); } - *(int32_t*)(&val->i64) = iv; - break; + int32_t tmpVal = (int32_t)iv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_UINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } else if (uv > UINT32_MAX) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); } - *(uint32_t*)(&val->i64) = uv; - break; + uint32_t tmpVal = (uint32_t)uv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } - val->i64 = iv; - break; + return func(&pCxt->msg, &iv, pSchema->bytes, param); } case TSDB_DATA_TYPE_UBIGINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } - *(uint64_t*)(&val->i64) = uv; - break; + return func(&pCxt->msg, &uv, pSchema->bytes, param); } case TSDB_DATA_TYPE_FLOAT: { double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - *(float*)(&val->i64) = dv; - break; + float tmpVal = (float)dv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_DOUBLE: { double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } - - *(double*)(&val->i64) = dv; - break; + return func(&pCxt->msg, &dv, pSchema->bytes, param); } case TSDB_DATA_TYPE_BINARY: { // Too long values will raise the invalid sql error message if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); - } - val->pData = strdup(pToken->z); - val->nData = pToken->n; - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - int32_t output = 0; - void* p = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - taosMemoryFree(p); - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); - } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); - taosMemoryFree(p); - return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z); - } - val->pData = p; - val->nData = output; - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: { - if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); - } - - val->i64 = iv; - break; - } - } - - return TSDB_CODE_SUCCESS; -} - -// pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); - SArray* tagName = taosArrayInit(8, TSDB_COL_NAME_LEN); - SToken sToken; - bool isParseBindParam = false; - bool isJson = false; - STag* pTag = NULL; - for (int i = 0; i < pCxt->tags.numOfBound; ++i) { - NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - - if (sToken.type == TK_NK_QUESTION) { - isParseBindParam = true; - if (NULL == pCxt->pStmtCb) { - code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); - break; + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - continue; - } - - if (isParseBindParam) { - code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); - break; - } - - SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // todo this can be optimize with parse column - code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg); - if (TSDB_CODE_SUCCESS == code) { - if (!isNullValue(pTagSchema->type, &sToken)) { - taosArrayPush(tagName, pTagSchema->name); - } - if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { - isJson = true; - if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z); - break; - } - if (isNullValue(pTagSchema->type, &sToken)) { - code = tTagNew(pTagVals, 1, true, &pTag); - } else { - code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); - } - } else { - STagVal val = {0}; - code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg); - if (TSDB_CODE_SUCCESS == code) { - taosArrayPush(pTagVals, &val); - } - } - } - if (TSDB_CODE_SUCCESS != code) { - break; + return func(&pCxt->msg, pToken->z, pToken->n, param); } - } - if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) { - code = tTagNew(pTagVals, 1, false, &pTag); - } - - if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - insBuildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName, - pCxt->pTableMeta->tableInfo.numOfTags); - pTag = NULL; - } - - for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { - STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); - if (IS_VAR_DATA_TYPE(p->type)) { - taosMemoryFreeClear(p->pData); - } - } - taosArrayDestroy(pTagVals); - taosArrayDestroy(tagName); - tTagFree(pTag); - return code; -} - -static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, int32_t tbNo, SName* pTableName, - const char* pName, int32_t len, STableMeta* pMeta) { - SVgroupInfo vg; - CHECK_CODE(getTableVgroup(pCxt, tbNo, pTableName, &vg)); - CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - - pMeta->uid = tbNo; - pMeta->vgId = vg.vgId; - pMeta->tableType = TSDB_CHILD_TABLE; - - STableMeta* pBackup = NULL; - if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); -} - -static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) { - SToken sToken; - int32_t expectRightParenthesis = 1; - while (1) { - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP == sToken.type) { - ++expectRightParenthesis; - } else if (TK_NK_RP == sToken.type && 0 == --expectRightParenthesis) { - break; - } - if (0 == sToken.n) { - return buildSyntaxErrMsg(&pCxt->msg, ") expected", NULL); + case TSDB_DATA_TYPE_NCHAR: { + return func(&pCxt->msg, pToken->z, pToken->n, param); } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t skipBoundColumns(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); } - -static int32_t ignoreBoundColumns(SInsertParseContext* pCxt) { - SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL}; - int32_t code = skipBoundColumns(&cxt); - pCxt->pSql = cxt.pSql; - return code; -} - -static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt); - -// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) -static int32_t ignoreAutoCreateTableClause(SInsertParseContext* pCxt) { - SToken sToken; - NEXT_TOKEN(pCxt->pSql, sToken); - SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL}; - int32_t code = skipUsingClause(&cxt); - pCxt->pSql = cxt.pSql; - return code; -} - -static int32_t parseTableOptions(SInsertParseContext* pCxt) { - do { - int32_t index = 0; - SToken sToken; - NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); - if (TK_TTL == sToken.type) { - pCxt->pSql += index; - NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - if (TK_NK_INTEGER != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z); - } - pCxt->createTblReq.ttl = taosStr2Int32(sToken.z, NULL, 10); - if (pCxt->createTblReq.ttl < 0) { - return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z); - } - } else if (TK_COMMENT == sToken.type) { - pCxt->pSql += index; - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_STRING != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "Invalid option comment", sToken.z); - } - if (sToken.n >= TSDB_TB_COMMENT_LEN) { - return buildSyntaxErrMsg(&pCxt->msg, "comment too long", sToken.z); - } - int32_t len = trimString(sToken.z, sToken.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); - pCxt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len); - if (NULL == pCxt->createTblReq.comment) { - return TSDB_CODE_OUT_OF_MEMORY; + case TSDB_DATA_TYPE_JSON: { + if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - pCxt->createTblReq.commentLen = len; - } else { - break; + return func(&pCxt->msg, pToken->z, pToken->n, param); } - } while (1); - return TSDB_CODE_SUCCESS; -} + case TSDB_DATA_TYPE_TIMESTAMP: { + int64_t tmpVal; + if (parseTime(pSql, pToken, timePrec, &tmpVal, &pCxt->msg) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); + } -// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) -static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* tbFName) { - int32_t len = strlen(tbFName); - STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len); - if (NULL != pMeta) { - CHECK_CODE(ignoreAutoCreateTableClause(pCxt)); - return cloneTableMeta(*pMeta, &pCxt->pTableMeta); + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } } - SToken sToken; - // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) - NEXT_TOKEN(pCxt->pSql, sToken); + return TSDB_CODE_FAILED; +} - SName sname; - CHECK_CODE(insCreateSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(&sname, dbFName); - strcpy(pCxt->sTableName, sname.tname); +static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, + int16_t timePrec, _row_append_fn_t func, void* param) { + int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) { + if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z); + } - CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname)); - if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { - return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); + return func(&pCxt->msg, NULL, 0, param); } - CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, tbNo, name, tbFName, len, pCxt->pTableMeta)); - - SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); - insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); - // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP == sToken.type) { - CHECK_CODE(parseBoundColumns(pCxt, &pCxt->tags, pTagsSchema)); - NEXT_TOKEN(pCxt->pSql, sToken); + if (TSDB_CODE_SUCCESS == code && IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); } - if (TK_TAGS != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z); - } - // pSql -> (tag1_value, ...) - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); - } - CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision, name->tname)); - NEXT_VALID_TOKEN(pCxt->pSql, sToken); - if (TK_NK_COMMA == sToken.type) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_TAGS_NOT_MATCHED); - } else if (TK_NK_RP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); + if (TSDB_CODE_SUCCESS == code) { + code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, func, param); } - return parseTableOptions(pCxt); + return code; } -static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, bool* gotRow, - char* tmpTokenBuf) { - SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlocks->rowBuilder; - STSRow* row = (STSRow*)(pDataBlocks->pData + pDataBlocks->size); // skip the SSubmitBlk header +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataBlocks* pDataBuf, bool* pGotRow, + SToken* pToken) { + SRowBuilder* pBuilder = &pDataBuf->rowBuilder; + STSRow* row = (STSRow*)(pDataBuf->pData + pDataBuf->size); // skip the SSubmitBlk header + SParsedDataColInfo* pCols = &pDataBuf->boundColumnInfo; + bool isParseBindParam = false; + SSchema* pSchemas = getTableColumnSchema(pDataBuf->pTableMeta); + SMemParam param = {.rb = pBuilder}; - tdSRowResetBuf(pBuilder, row); - - bool isParseBindParam = false; - SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta); - SMemParam param = {.rb = pBuilder}; - SToken sToken = {0}; + int32_t code = tdSRowResetBuf(pBuilder, row); // 1. set the parsed value from sql string - for (int i = 0; i < spd->numOfBound; ++i) { - NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pSchema = &schema[spd->boundColumns[i]]; + for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { + NEXT_TOKEN_WITH_PREV(*pSql, *pToken); + SSchema* pSchema = &pSchemas[pCols->boundColumns[i]]; - if (sToken.type == TK_NK_QUESTION) { + if (pToken->type == TK_NK_QUESTION) { isParseBindParam = true; - if (NULL == pCxt->pStmtCb) { - return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + if (NULL == pCxt->pComCxt->pStmtCb) { + code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pToken->z); } - continue; } - if (TK_NK_RP == sToken.type) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + if (TSDB_CODE_SUCCESS == code && TK_NK_RP == pToken->type) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); } - if (isParseBindParam) { - return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values"); + if (TSDB_CODE_SUCCESS == code && isParseBindParam) { + code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values"); } - param.schema = pSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); - CHECK_CODE( - parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, insMemRowAppend, ¶m, &pCxt->msg)); + if (TSDB_CODE_SUCCESS == code) { + param.schema = pSchema; + insGetSTSRowAppendInfo(pBuilder->rowType, pCols, i, ¶m.toffset, ¶m.colIdx); + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pDataBuf->pTableMeta).precision, insMemRowAppend, + ¶m); + } - if (i < spd->numOfBound - 1) { - NEXT_VALID_TOKEN(pCxt->pSql, sToken); - if (TK_NK_COMMA != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, ", expected", sToken.z); + if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { + NEXT_VALID_TOKEN(*pSql, *pToken); + if (TK_NK_COMMA != pToken->type) { + code = buildSyntaxErrMsg(&pCxt->msg, ", expected", pToken->z); } } } - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlocks, (const char*)&tsKey); + if (TSDB_CODE_SUCCESS == code) { + TSKEY tsKey = TD_ROW_KEY(row); + code = insCheckTimestamp(pDataBuf, (const char*)&tsKey); + } - if (!isParseBindParam) { + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + if ((pCols->numOfBound < pCols->numOfCols) && TD_IS_TP_ROW(row)) { pBuilder->hasNone = true; } tdSRowEnd(pBuilder); - *gotRow = true; + *pGotRow = true; #ifdef TD_DEBUG_PRINT_ROW STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); @@ -981,8 +1224,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, #endif } - // *len = pBuilder->extendedRowSize; - return TSDB_CODE_SUCCESS; + return code; } static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, int32_t* numOfRows) { @@ -1014,84 +1256,81 @@ static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows) { - STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); - - (*numOfRows) = 0; - // char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - SToken sToken; - while (1) { +static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, + int32_t maxRows, int32_t* pNumOfRows, SToken* pToken) { + int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); + + int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); + (*pNumOfRows) = 0; + while (TSDB_CODE_SUCCESS == code) { int32_t index = 0; - NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); - if (TK_NK_LP != sToken.type) { + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, *pToken, index); + if (TK_NK_LP != pToken->type) { break; } - pCxt->pSql += index; + pStmt->pSql += index; - if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { - int32_t tSize; - CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize)); - ASSERT(tSize >= maxRows); - maxRows = tSize; + if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { + code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); } bool gotRow = false; - CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, pCxt->tmpTokenBuf)); - if (gotRow) { - pDataBlock->size += extendedRowSize; // len; + if (TSDB_CODE_SUCCESS == code) { + code = parseOneRow(pCxt, &pStmt->pSql, pDataBuf, &gotRow, pToken); } - NEXT_VALID_TOKEN(pCxt->pSql, sToken); - if (TK_NK_COMMA == sToken.type) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); - } else if (TK_NK_RP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z); + if (TSDB_CODE_SUCCESS == code) { + NEXT_VALID_TOKEN(pStmt->pSql, *pToken); + if (TK_NK_COMMA == pToken->type) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + } else if (TK_NK_RP != pToken->type) { + code = buildSyntaxErrMsg(&pCxt->msg, ") expected", pToken->z); + } } - if (gotRow) { - (*numOfRows)++; + if (TSDB_CODE_SUCCESS == code && gotRow) { + pDataBuf->size += extendedRowSize; + (*pNumOfRows)++; } } - if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { - return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && + (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } - return TSDB_CODE_SUCCESS; + return code; } -static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* dataBuf) { - int32_t maxNumOfRows; - CHECK_CODE(allocateMemIfNeed(dataBuf, insGetExtendedRowSize(dataBuf), &maxNumOfRows)); - +// VALUES (field1_value, ...) [(field1_value2, ...) ...] +static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, + SToken* pToken) { + int32_t maxNumOfRows = 0; int32_t numOfRows = 0; - CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows)); - - SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, - "too many rows in sql, total number of rows should be less than INT32_MAX"); + int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + if (TSDB_CODE_SUCCESS == code) { + code = parseValues(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows, pToken); } - - dataBuf->numOfTables = 1; - pCxt->totalNum += numOfRows; - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) { + code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); + } + if (TSDB_CODE_SUCCESS == code) { + pDataBuf->numOfTables = 1; + pStmt->totalRowsNum += numOfRows; + pStmt->totalTbNum += 1; + TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); + } + return code; } -static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataBlocks* pDataBlock, int maxRows, - int32_t* numOfRows) { - STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); +static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, + int maxRows, int32_t* pNumOfRows) { + int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); - (*numOfRows) = 0; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); + (*pNumOfRows) = 0; char* pLine = NULL; int64_t readLen = 0; - while ((readLen = taosGetLineFile(fp, &pLine)) != -1) { + while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pStmt->fp, &pLine)) != -1) { if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) { pLine[--readLen] = '\0'; } @@ -1100,588 +1339,516 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB continue; } - if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { - int32_t tSize; - CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize)); - ASSERT(tSize >= maxRows); - maxRows = tSize; + if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { + code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); } - strtolower(pLine, pLine); - char* pRawSql = pCxt->pSql; - pCxt->pSql = pLine; - bool gotRow = false; - int32_t code = parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, tmpTokenBuf); - if (TSDB_CODE_SUCCESS != code) { - pCxt->pSql = pRawSql; - return code; + bool gotRow = false; + if (TSDB_CODE_SUCCESS == code) { + SToken token; + strtolower(pLine, pLine); + code = parseOneRow(pCxt, (const char**)&pLine, pDataBuf, &gotRow, &token); } - if (gotRow) { - pDataBlock->size += extendedRowSize; // len; - (*numOfRows)++; + + if (TSDB_CODE_SUCCESS == code && gotRow) { + pDataBuf->size += extendedRowSize; + (*pNumOfRows)++; } - pCxt->pSql = pRawSql; - if (pDataBlock->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && pDataBuf->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { break; } } - if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { - return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && + (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } - return TSDB_CODE_SUCCESS; + return code; } -static int32_t parseDataFromFileAgain(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, - STableDataBlocks* dataBuf) { - int32_t maxNumOfRows; - CHECK_CODE(allocateMemIfNeed(dataBuf, insGetExtendedRowSize(dataBuf), &maxNumOfRows)); - +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) { + int32_t maxNumOfRows = 0; int32_t numOfRows = 0; - CHECK_CODE(parseCsvFile(pCxt, pCxt->pComCxt->csvCxt.fp, dataBuf, maxNumOfRows, &numOfRows)); - - SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, - "too many rows in sql, total number of rows should be less than INT32_MAX"); + int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + if (TSDB_CODE_SUCCESS == code) { + code = parseCsvFile(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows); } - - if (!taosEOFFile(pCxt->pComCxt->csvCxt.fp)) { - pCxt->pComCxt->needMultiParse = true; - pCxt->pComCxt->csvCxt.tableNo = tableNo; - memcpy(&pCxt->pComCxt->csvCxt.tableName, pTableName, sizeof(SName)); - pCxt->pComCxt->csvCxt.pLastSqlPos = pCxt->pSql; + if (TSDB_CODE_SUCCESS == code) { + code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); + } + if (TSDB_CODE_SUCCESS == code) { + pDataBuf->numOfTables = 1; + pStmt->totalRowsNum += numOfRows; + pStmt->totalTbNum += 1; + TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); + if (taosEOFFile(pStmt->fp)) { + taosCloseFile(&pStmt->fp); + } else { + parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); + } } - - dataBuf->numOfTables = 1; - pCxt->totalNum += numOfRows; return TSDB_CODE_SUCCESS; } -static int32_t parseDataFromFile(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, SToken filePath, - STableDataBlocks* dataBuf) { +static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pFilePath, + STableDataBlocks* pDataBuf) { char filePathStr[TSDB_FILENAME_LEN] = {0}; - if (TK_NK_STRING == filePath.type) { - trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr)); + if (TK_NK_STRING == pFilePath->type) { + trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); } else { - strncpy(filePathStr, filePath.z, filePath.n); + strncpy(filePathStr, pFilePath->z, pFilePath->n); } - pCxt->pComCxt->csvCxt.fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM); - if (NULL == pCxt->pComCxt->csvCxt.fp) { + pStmt->fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM); + if (NULL == pStmt->fp) { return TAOS_SYSTEM_ERROR(errno); } - return parseDataFromFileAgain(pCxt, tableNo, pTableName, dataBuf); + return parseDataFromFileImpl(pCxt, pStmt, pDataBuf); } -static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { - if (!pCxt->pComCxt->needMultiParse) { - taosCloseFile(&pCxt->pComCxt->csvCxt.fp); +static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, + SToken* pToken) { + NEXT_TOKEN(pStmt->pSql, *pToken); + if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { + return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); } - taosMemoryFreeClear(pCxt->pTableMeta); - destroyBoundColumnInfo(&pCxt->tags); - tdDestroySVCreateTbReq(&pCxt->createTblReq); + return parseDataFromFile(pCxt, pStmt, pToken, pDataBuf); } -static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } - -static void destroyInsertParseContext(SInsertParseContext* pCxt) { - destroyInsertParseContextForTable(pCxt); - taosHashCleanup(pCxt->pVgroupsHashObj); - taosHashCleanup(pCxt->pSubTableHashObj); - taosHashCleanup(pCxt->pTableNameHashObj); - taosHashCleanup(pCxt->pDbFNameHashObj); - - insDestroyBlockHashmap(pCxt->pTableBlockHashObj); - insDestroyBlockArrayList(pCxt->pVgDataBlocks); +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) { + SToken token; + NEXT_TOKEN(pStmt->pSql, token); + switch (token.type) { + case TK_VALUES: + return parseValuesClause(pCxt, pStmt, pDataBuf, &token); + case TK_FILE: + return parseFileClause(pCxt, pStmt, pDataBuf, &token); + default: + break; + } + return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -static int32_t parseTableName(SInsertParseContext* pCxt, SToken* pTbnameToken, SName* pName, char* pDbFName, - char* pTbFName) { - int32_t code = insCreateSName(pName, pTbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); - if (TSDB_CODE_SUCCESS == code) { - tNameExtractFullName(pName, pTbFName); - code = taosHashPut(pCxt->pTableNameHashObj, pTbFName, strlen(pTbFName), pName, sizeof(SName)); - } +// input pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + STableDataBlocks* pDataBuf = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pDataBuf); if (TSDB_CODE_SUCCESS == code) { - tNameGetFullDbName(pName, pDbFName); - code = taosHashPut(pCxt->pDbFNameHashObj, pDbFName, strlen(pDbFName), pDbFName, TSDB_DB_FNAME_LEN); + code = parseDataClause(pCxt, pStmt, pDataBuf); } return code; } -// tb_name -// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] -// [(field1_name, ...)] -// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -// [...]; -static int32_t parseInsertBody(SInsertParseContext* pCxt) { - int32_t tbNum = 0; - SName name; - char tbFName[TSDB_TABLE_FNAME_LEN]; - char dbFName[TSDB_DB_FNAME_LEN]; - bool autoCreateTbl = false; - - // for each table - while (1) { - SToken sToken; - char* tbName = NULL; - - // pSql -> tb_name ... - NEXT_TOKEN(pCxt->pSql, sToken); - - // no data in the sql string anymore. - if (sToken.n == 0) { - if (sToken.type && pCxt->pSql[0]) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z); - } +// input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... +static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = parseInsertTableClauseBottom(pCxt, pStmt); + } + return code; +} - if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && - !pCxt->pComCxt->needMultiParse) { - return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); - } - break; +static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName, + bool* pHasData) { + // no data in the sql string anymore. + if (0 == pTbName->n) { + if (0 != pTbName->type && '\0' != pStmt->pSql[0]) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", pTbName->z); } - if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && tbNum > 0) { - return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt"); + if (0 == pStmt->totalRowsNum && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); } - destroyInsertParseContextForTable(pCxt); - - if (TK_NK_QUESTION == sToken.type) { - if (pCxt->pStmtCb) { - CHECK_CODE((*pCxt->pStmtCb->getTbNameFn)(pCxt->pStmtCb->pStmt, &tbName)); - - sToken.z = tbName; - sToken.n = strlen(tbName); - } else { - return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); - } - } + *pHasData = false; + return TSDB_CODE_SUCCESS; + } - SToken tbnameToken = sToken; - NEXT_TOKEN(pCxt->pSql, sToken); + if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && pStmt->totalTbNum > 0) { + return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt"); + } - if (!pCxt->pComCxt->async || TK_USING == sToken.type) { - CHECK_CODE(parseTableName(pCxt, &tbnameToken, &name, dbFName, tbFName)); + if (TK_NK_QUESTION == pTbName->type) { + if (NULL == pCxt->pComCxt->pStmtCb) { + return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pTbName->z); } - bool existedUsing = false; - // USING clause - if (TK_USING == sToken.type) { - existedUsing = true; - CHECK_CODE(parseUsingClause(pCxt, tbNum, &name, tbFName)); - NEXT_TOKEN(pCxt->pSql, sToken); - autoCreateTbl = true; + char* tbName = NULL; + int32_t code = (*pCxt->pComCxt->pStmtCb->getTbNameFn)(pCxt->pComCxt->pStmtCb->pStmt, &tbName); + if (TSDB_CODE_SUCCESS == code) { + pTbName->z = tbName; + pTbName->n = strlen(tbName); + } else { + return code; } + } - char* pBoundColsStart = NULL; - if (TK_NK_LP == sToken.type) { - // pSql -> field1_name, ...) - pBoundColsStart = pCxt->pSql; - CHECK_CODE(ignoreBoundColumns(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); - } + *pHasData = true; + return TSDB_CODE_SUCCESS; +} - if (TK_USING == sToken.type) { - if (pCxt->pComCxt->async) { - CHECK_CODE(parseTableName(pCxt, &tbnameToken, &name, dbFName, tbFName)); - } - CHECK_CODE(parseUsingClause(pCxt, tbNum, &name, tbFName)); - NEXT_TOKEN(pCxt->pSql, sToken); - autoCreateTbl = true; - } else if (!existedUsing) { - CHECK_CODE(getTableMeta(pCxt, tbNum, &name)); - if (TSDB_SUPER_TABLE == pCxt->pTableMeta->tableType) { - return buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); - } - } +static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); + if (NULL == tags) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - STableDataBlocks* dataBuf = NULL; - if (pCxt->pComCxt->async) { - CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid, - sizeof(pCxt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), - getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, - &pCxt->createTblReq)); - } else { - CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, - &dataBuf, NULL, &pCxt->createTblReq)); - } + SStmtCallback* pStmtCb = pCxt->pComCxt->pStmtCb; + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + char stbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->usingTableName, stbFName); + int32_t code = + (*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, tbFName, '\0' != pStmt->usingTableName.tname[0], + pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj, stbFName); - if (NULL != pBoundColsStart) { - char* pCurrPos = pCxt->pSql; - pCxt->pSql = pBoundColsStart; - CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); - pCxt->pSql = pCurrPos; - } + memset(&pCxt->tags, 0, sizeof(pCxt->tags)); + pStmt->pVgroupsHashObj = NULL; + pStmt->pTableBlockHashObj = NULL; + return code; +} - if (TK_VALUES == sToken.type) { - // pSql -> (field1_value, ...) [(field1_value2, ...) ...] - CHECK_CODE(parseValuesClause(pCxt, dataBuf)); - TSDB_QUERY_SET_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_INSERT); +static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + return setStmtInfo(pCxt, pStmt); + } - tbNum++; - continue; - } + // merge according to vgId + int32_t code = TSDB_CODE_SUCCESS; + if (taosHashGetSize(pStmt->pTableBlockHashObj) > 0) { + code = insMergeTableDataBlocks(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); + } + if (TSDB_CODE_SUCCESS == code) { + code = insBuildOutput(pStmt); + } + return code; +} - // FILE csv_file_path - if (TK_FILE == sToken.type) { - // pSql -> csv_file_path - NEXT_TOKEN(pCxt->pSql, sToken); - if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) { - return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); - } - CHECK_CODE(parseDataFromFile(pCxt, tbNum, &name, sToken, dataBuf)); - pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT; +static void destroyEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + destroyBoundColumnInfo(&pCxt->tags); + taosMemoryFreeClear(pStmt->pTableMeta); + tdDestroySVCreateTbReq(&pStmt->createTblReq); +} - tbNum++; - if (!pCxt->pComCxt->needMultiParse) { - continue; - } else { - parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); - break; - } +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + int32_t code = TSDB_CODE_SUCCESS; + bool hasData = true; + // for each table + while (TSDB_CODE_SUCCESS == code && hasData && !pCxt->missCache) { + destroyEnvPreTable(pCxt, pStmt); + // pStmt->pSql -> tb_name ... + NEXT_TOKEN(pStmt->pSql, token); + code = checkTableClauseFirstToken(pCxt, pStmt, &token, &hasData); + if (TSDB_CODE_SUCCESS == code && hasData) { + code = parseInsertTableClause(pCxt, pStmt, &token); } - - return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } - parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pStmt->totalRowsNum); - if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { - SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); - if (NULL == tags) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, - pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj, pCxt->sTableName); + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = parseInsertBodyBottom(pCxt, pStmt); + } + return code; +} - memset(&pCxt->tags, 0, sizeof(pCxt->tags)); - pCxt->pVgroupsHashObj = NULL; - pCxt->pTableBlockHashObj = NULL; +static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } - return TSDB_CODE_SUCCESS; +static int32_t createVnodeModifOpStmt(SParseContext* pCxt, SNode** pOutput) { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if (NULL == pStmt) { + return TSDB_CODE_OUT_OF_MEMORY; } - // merge according to vgId - if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(insMergeTableDataBlocks(pCxt->pTableBlockHashObj, &pCxt->pVgDataBlocks)); - } - return insBuildOutput(pCxt); -} - -static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) { - STableDataBlocks* dataBuf = NULL; - CHECK_CODE(getTableMeta(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName)); - CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid, sizeof(pCxt->pTableMeta->uid), - TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), - getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, - &pCxt->createTblReq)); - CHECK_CODE(parseDataFromFileAgain(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName, dataBuf)); - if (taosEOFFile(pCxt->pComCxt->csvCxt.fp)) { - CHECK_CODE(parseInsertBody(pCxt)); - pCxt->pComCxt->needMultiParse = false; - return TSDB_CODE_SUCCESS; + if (pCxt->pStmtCb) { + TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } - parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); - // merge according to vgId - if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(insMergeTableDataBlocks(pCxt->pTableBlockHashObj, &pCxt->pVgDataBlocks)); + pStmt->pSql = pCxt->pSql; + pStmt->freeHashFunc = insDestroyBlockHashmap; + pStmt->freeArrayFunc = insDestroyBlockArrayList; + + pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pStmt->pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + pStmt->pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + pStmt->pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + pStmt->pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + if (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj || NULL == pStmt->pSubTableHashObj || + NULL == pStmt->pTableNameHashObj || NULL == pStmt->pDbFNameHashObj) { + nodesDestroyNode((SNode*)pStmt); + return TSDB_CODE_OUT_OF_MEMORY; } - return insBuildOutput(pCxt); + + taosHashSetFreeFp(pStmt->pSubTableHashObj, destroySubTableHashElem); + + *pOutput = (SNode*)pStmt; + return TSDB_CODE_SUCCESS; } -// INSERT INTO -// tb_name -// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] -// [(field1_name, ...)] -// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -// [...]; -int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) { - SInsertParseContext context = { - .pComCxt = pContext, - .pSql = pContext->needMultiParse ? (char*)pContext->csvCxt.pLastSqlPos : (char*)pContext->pSql, - .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pTableMeta = NULL, - .createTblReq = {0}, - .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), - .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), - .pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), - .totalNum = 0, - .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), - .pStmtCb = pContext->pStmtCb, - .pMetaCache = pMetaCache, - .memElapsed = 0, - .parRowElapsed = 0}; - - if (pContext->pStmtCb && *pQuery) { - (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, - &context.pTableBlockHashObj); - if (NULL == context.pVgroupsHashObj) { - context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - if (NULL == context.pTableBlockHashObj) { - context.pTableBlockHashObj = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - } - } else { - context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - context.pTableBlockHashObj = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); +static int32_t createInsertQuery(SParseContext* pCxt, SQuery** pOutput) { + SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + return TSDB_CODE_OUT_OF_MEMORY; } - if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj || - NULL == context.pTableNameHashObj || NULL == context.pDbFNameHashObj || NULL == context.pOutput) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - taosHashSetFreeFp(context.pSubTableHashObj, destroySubTableHashElem); + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; - if (pContext->pStmtCb) { - TSDB_QUERY_SET_TYPE(context.pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT); + int32_t code = createVnodeModifOpStmt(pCxt, &pQuery->pRoot); + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pQuery; + } else { + nodesDestroyNode((SNode*)pQuery); } + return code; +} - if (NULL == *pQuery) { - *pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == *pQuery) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { - nodesDestroyNode((*pQuery)->pRoot); +static int32_t checkAuthFromMetaData(const SArray* pUsers) { + if (1 != taosArrayGetSize(pUsers)) { + return TSDB_CODE_FAILED; } - (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; - (*pQuery)->haveResultSet = false; - (*pQuery)->msgType = TDMT_VND_SUBMIT; - (*pQuery)->pRoot = (SNode*)context.pOutput; + SMetaRes* pRes = taosArrayGet(pUsers, 0); + if (TSDB_CODE_SUCCESS == pRes->code) { + return (*(bool*)pRes->pRes) ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED; + } + return pRes->code; +} - if (NULL == (*pQuery)->pTableList) { - (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); - if (NULL == (*pQuery)->pTableList) { +static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMeta) { + if (1 != taosArrayGetSize(pTables)) { + return TSDB_CODE_FAILED; + } + SMetaRes* pRes = taosArrayGet(pTables, 0); + if (TSDB_CODE_SUCCESS == pRes->code) { + *pMeta = tableMetaDup((const STableMeta*)pRes->pRes); + if (NULL == *pMeta) { return TSDB_CODE_OUT_OF_MEMORY; } } + return pRes->code; +} - if (NULL == (*pQuery)->pDbList) { - (*pQuery)->pDbList = taosArrayInit(taosHashGetSize(context.pDbFNameHashObj), TSDB_DB_FNAME_LEN); - if (NULL == (*pQuery)->pDbList) { - return TSDB_CODE_OUT_OF_MEMORY; - } +static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpStmt* pStmt, bool isStb) { + if (1 != taosArrayGetSize(pTables)) { + return TSDB_CODE_FAILED; } - int32_t code = TSDB_CODE_SUCCESS; - if (!context.pComCxt->needMultiParse) { - code = skipInsertInto(&context.pSql, &context.msg); - if (TSDB_CODE_SUCCESS == code) { - code = parseInsertBody(&context); - } - } else { - code = parseInsertBodyAgain(&context); + SMetaRes* pRes = taosArrayGet(pTables, 0); + if (TSDB_CODE_SUCCESS != pRes->code) { + return pRes->code; } - if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) { - SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL); - while (NULL != pTable) { - taosArrayPush((*pQuery)->pTableList, pTable); - pTable = taosHashIterate(context.pTableNameHashObj, pTable); - } + SVgroupInfo* pVg = pRes->pRes; + if (isStb) { + pStmt->pTableMeta->vgId = pVg->vgId; + } + return taosHashPut(pStmt->pVgroupsHashObj, (const char*)&pVg->vgId, sizeof(pVg->vgId), (char*)pVg, + sizeof(SVgroupInfo)); +} - char* pDb = taosHashIterate(context.pDbFNameHashObj, NULL); - while (NULL != pDb) { - taosArrayPush((*pQuery)->pDbList, pDb); - pDb = taosHashIterate(context.pDbFNameHashObj, pDb); - } +static int32_t getTableSchemaFromMetaData(const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt, bool isStb) { + int32_t code = checkAuthFromMetaData(pMetaData->pUser); + if (TSDB_CODE_SUCCESS == code) { + code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); } - if (pContext->pStmtCb) { - context.pVgroupsHashObj = NULL; - context.pTableBlockHashObj = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); } - destroyInsertParseContext(&context); return code; } -// pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t skipValuesClause(SInsertParseSyntaxCxt* pCxt) { - int32_t numOfRows = 0; - SToken sToken; - while (1) { - int32_t index = 0; - NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); - if (TK_NK_LP != sToken.type) { - break; +static int32_t setVnodeModifOpStmt(SParseContext* pCxt, const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt) { + if (pCxt->pStmtCb) { + (*pCxt->pStmtCb->getExecInfoFn)(pCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj); + if (NULL == pStmt->pVgroupsHashObj) { + pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + } + if (NULL == pStmt->pTableBlockHashObj) { + pStmt->pTableBlockHashObj = + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - pCxt->pSql += index; - CHECK_CODE(skipParentheses(pCxt)); - ++numOfRows; + TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } - if (0 == numOfRows) { - return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + + if (pStmt->usingTableProcessing) { + return getTableSchemaFromMetaData(pMetaData, pStmt, true); } - return TSDB_CODE_SUCCESS; + return getTableSchemaFromMetaData(pMetaData, pStmt, false); } -static int32_t skipTagsClause(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); } - -static int32_t skipTableOptions(SInsertParseSyntaxCxt* pCxt) { - do { - int32_t index = 0; - SToken sToken; - NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); - if (TK_TTL == sToken.type || TK_COMMENT == sToken.type) { - pCxt->pSql += index; - NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - } else { - break; - } - } while (1); - return TSDB_CODE_SUCCESS; +static int32_t initInsertQuery(SParseContext* pCxt, const SMetaData* pMetaData, SQuery** pQuery) { + if (NULL == *pQuery) { + return createInsertQuery(pCxt, pQuery); + } + return setVnodeModifOpStmt(pCxt, pMetaData, (SVnodeModifOpStmt*)(*pQuery)->pRoot); } -// pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) -static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) { - SToken sToken; - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP == sToken.type) { - CHECK_CODE(skipBoundColumns(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); +static int32_t setRefreshMate(SQuery* pQuery) { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + SName* pTable = taosHashIterate(pStmt->pTableNameHashObj, NULL); + while (NULL != pTable) { + taosArrayPush(pQuery->pTableList, pTable); + pTable = taosHashIterate(pStmt->pTableNameHashObj, pTable); } - if (TK_TAGS != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z); - } - // pSql -> (tag1_value, ...) - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); + char* pDb = taosHashIterate(pStmt->pDbFNameHashObj, NULL); + while (NULL != pDb) { + taosArrayPush(pQuery->pDbList, pDb); + pDb = taosHashIterate(pStmt->pDbFNameHashObj, pDb); } - CHECK_CODE(skipTagsClause(pCxt)); - CHECK_CODE(skipTableOptions(pCxt)); return TSDB_CODE_SUCCESS; } -static int32_t collectTableMetaKey(SInsertParseSyntaxCxt* pCxt, bool isStable, int32_t tableNo, SToken* pTbToken) { - SName name = {0}; - CHECK_CODE(insCreateSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - CHECK_CODE(reserveTableMetaInCacheForInsert(&name, isStable ? CATALOG_REQ_TYPE_META : CATALOG_REQ_TYPE_BOTH, tableNo, - pCxt->pMetaCache)); - return TSDB_CODE_SUCCESS; +// INSERT INTO +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = skipInsertInto(&pStmt->pSql, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + code = parseInsertBody(pCxt, pStmt); + } + return code; } -static int32_t checkTableName(const char* pTableName, SMsgBuf* pMsgBuf) { - if (NULL != strchr(pTableName, '.')) { - return generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); +static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + STableDataBlocks* pDataBuf = NULL; + int32_t code = getTableDataBlocks(pCxt, pStmt, &pDataBuf); + if (TSDB_CODE_SUCCESS == code) { + code = parseDataFromFileImpl(pCxt, pStmt, pDataBuf); } - return TSDB_CODE_SUCCESS; -} -static int32_t collectAutoCreateTableMetaKey(SInsertParseSyntaxCxt* pCxt, int32_t tableNo, SToken* pTbToken) { - SName name = {0}; - CHECK_CODE(insCreateSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - CHECK_CODE(checkTableName(name.tname, &pCxt->msg)); - CHECK_CODE(reserveTableMetaInCacheForInsert(&name, CATALOG_REQ_TYPE_VGROUP, tableNo, pCxt->pMetaCache)); - return TSDB_CODE_SUCCESS; + parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pStmt->totalRowsNum); + + if (TSDB_CODE_SUCCESS == code) { + if (pStmt->fileProcessing) { + code = parseInsertBodyBottom(pCxt, pStmt); + } else { + code = parseInsertBody(pCxt, pStmt); + } + } + + return code; } -static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) { - bool hasData = false; - int32_t tableNo = 0; - // for each table - while (1) { - SToken sToken; +static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = parseInsertTableClauseBottom(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = parseInsertBody(pCxt, pStmt); + } + return code; +} - // pSql -> tb_name ... - NEXT_TOKEN(pCxt->pSql, sToken); +static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + if (pStmt->pSql == pCxt->pComCxt->pSql || NULL != pCxt->pComCxt->pStmtCb) { + return parseInsertSqlFromStart(pCxt, pStmt); + } - // no data in the sql string anymore. - if (sToken.n == 0) { - if (sToken.type && pCxt->pSql[0]) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z); - } + if (pStmt->fileProcessing) { + return parseInsertSqlFromCsv(pCxt, pStmt); + } - if (!hasData) { - return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); - } - break; - } + return parseInsertSqlFromTable(pCxt, pStmt); +} - hasData = false; +static int32_t buildInsertTableReq(SName* pName, SArray** pTables) { + *pTables = taosArrayInit(1, sizeof(SName)); + if (NULL == *pTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } - SToken tbnameToken = sToken; - NEXT_TOKEN(pCxt->pSql, sToken); + taosArrayPush(*pTables, pName); + return TSDB_CODE_SUCCESS; +} - bool existedUsing = false; - // USING clause - if (TK_USING == sToken.type) { - existedUsing = true; - CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, tableNo, &tbnameToken)); - NEXT_TOKEN(pCxt->pSql, sToken); - CHECK_CODE(collectTableMetaKey(pCxt, true, tableNo, &sToken)); - CHECK_CODE(skipUsingClause(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); +static int32_t buildInsertDbReq(SName* pName, SArray** pDbs) { + if (NULL == *pDbs) { + *pDbs = taosArrayInit(1, sizeof(STablesReq)); + if (NULL == *pDbs) { + return TSDB_CODE_OUT_OF_MEMORY; } + } - if (TK_NK_LP == sToken.type) { - // pSql -> field1_name, ...) - CHECK_CODE(skipBoundColumns(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); - } + STablesReq req = {0}; + tNameGetFullDbName(pName, req.dbFName); + buildInsertTableReq(pName, &req.pTables); + taosArrayPush(*pDbs, &req); - if (TK_USING == sToken.type && !existedUsing) { - existedUsing = true; - CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, tableNo, &tbnameToken)); - NEXT_TOKEN(pCxt->pSql, sToken); - CHECK_CODE(collectTableMetaKey(pCxt, true, tableNo, &sToken)); - CHECK_CODE(skipUsingClause(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); - } else if (!existedUsing) { - CHECK_CODE(collectTableMetaKey(pCxt, false, tableNo, &tbnameToken)); - } + return TSDB_CODE_SUCCESS; +} - ++tableNo; +static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** pUserAuth) { + *pUserAuth = taosArrayInit(1, sizeof(SUserAuthInfo)); + if (NULL == *pUserAuth) { + return TSDB_CODE_OUT_OF_MEMORY; + } - if (TK_VALUES == sToken.type) { - // pSql -> (field1_value, ...) [(field1_value2, ...) ...] - CHECK_CODE(skipValuesClause(pCxt)); - hasData = true; - continue; - } + SUserAuthInfo userAuth = {.type = AUTH_TYPE_WRITE}; + snprintf(userAuth.user, sizeof(userAuth.user), "%s", pUser); + tNameGetFullDbName(pName, userAuth.dbFName); + taosArrayPush(*pUserAuth, &userAuth); - // FILE csv_file_path - if (TK_FILE == sToken.type) { - // pSql -> csv_file_path - NEXT_TOKEN(pCxt->pSql, sToken); - if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) { - return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); - } - hasData = true; - continue; + return TSDB_CODE_SUCCESS; +} + +static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SCatalogReq* pCatalogReq) { + int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser); + if (TSDB_CODE_SUCCESS == code) { + if (0 == pStmt->usingTableName.type) { + code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableMeta); + } else { + code = buildInsertDbReq(&pStmt->usingTableName, &pCatalogReq->pTableMeta); } + } + if (TSDB_CODE_SUCCESS == code) { + code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableHash); + } + return code; +} - return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); +static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) { + if (pCxt->missCache) { + pQuery->execStage = QUERY_EXEC_STAGE_PARSE; + return buildInsertCatalogReq(pCxt, (SVnodeModifOpStmt*)pQuery->pRoot, pCatalogReq); } + pQuery->execStage = QUERY_EXEC_STAGE_SCHEDULE; return TSDB_CODE_SUCCESS; } -int32_t parseInsertSyntax(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) { - SInsertParseSyntaxCxt context = {.pComCxt = pContext, - .pSql = (char*)pContext->pSql, - .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pMetaCache = pMetaCache}; - int32_t code = skipInsertInto(&context.pSql, &context.msg); +int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData) { + SInsertParseContext context = { + .pComCxt = pCxt, + .msg = {.buf = pCxt->pMsg, .len = pCxt->msgLen}, + }; + + int32_t code = initInsertQuery(pCxt, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = parseInsertBodySyntax(&context); + code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot); } if (TSDB_CODE_SUCCESS == code) { - *pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == *pQuery) { - return TSDB_CODE_OUT_OF_MEMORY; - } + code = setNextStageInfo(&context, *pQuery, pCatalogReq); + } + if ((TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) && + QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { + code = setRefreshMate(*pQuery); } + destroyBoundColumnInfo(&context.tags); return code; } diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index f85ceccf6e..73557c067e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -30,22 +30,17 @@ typedef struct SKvParam { } SKvParam; int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { - SVnodeModifOpStmt* modifyNode = (SVnodeModifOpStmt*)pQuery->pRoot; - int32_t code = 0; - SInsertParseContext insertCtx = { - .pVgroupsHashObj = pVgHash, - .pTableBlockHashObj = pBlockHash, - .pOutput = (SVnodeModifOpStmt*)pQuery->pRoot, - }; + SVnodeModifOpStmt* modifyNode = (SVnodeModifOpStmt*)pQuery->pRoot; + int32_t code = 0; // merge according to vgId - if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) { - CHECK_CODE(insMergeTableDataBlocks(insertCtx.pTableBlockHashObj, &insertCtx.pVgDataBlocks)); + if (taosHashGetSize(pBlockHash) > 0) { + CHECK_CODE(insMergeTableDataBlocks(pBlockHash, &modifyNode->pVgDataBlocks)); } - CHECK_CODE(insBuildOutput(&insertCtx)); + CHECK_CODE(insBuildOutput(modifyNode)); - insDestroyBlockArrayList(insertCtx.pVgDataBlocks); + insDestroyBlockArrayList(modifyNode->pVgDataBlocks); return TSDB_CODE_SUCCESS; } @@ -222,11 +217,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX"); - } - - return TSDB_CODE_SUCCESS; + return insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf); } int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, @@ -308,10 +299,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu pDataBlock->size += extendedRowSize * bind->num; SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, - "too many rows in sql, total number of rows should be less than INT32_MAX"); - } + CHECK_CODE(insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf)); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 570a6f9859..a0a24d3d31 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -110,18 +110,17 @@ void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo* spd, col_id_t i } } -int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows) { +int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows, SMsgBuf* pMsg) { pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid); pBlocks->uid = dataBuf->pTableMeta->uid; pBlocks->sversion = dataBuf->pTableMeta->sversion; pBlocks->schemaLen = dataBuf->createTbReqLen; if (pBlocks->numOfRows + numOfRows >= INT32_MAX) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else { - pBlocks->numOfRows += numOfRows; - return TSDB_CODE_SUCCESS; + return buildInvalidOperationMsg(pMsg, "too many rows in sql, total number of rows should be less than INT32_MAX"); } + pBlocks->numOfRows += numOfRows; + return TSDB_CODE_SUCCESS; } void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { @@ -271,12 +270,8 @@ void insDestroyDataBlock(STableDataBlocks* pDataBlock) { } taosMemoryFreeClear(pDataBlock->pData); - // if (!pDataBlock->cloned) { - // free the refcount for metermeta taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - // } taosMemoryFreeClear(pDataBlock); } @@ -312,20 +307,6 @@ int32_t insGetDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, in return TSDB_CODE_SUCCESS; } -#if 0 -static int32_t getRowExpandSize(STableMeta* pTableMeta) { - int32_t result = TD_ROW_HEAD_LEN - sizeof(TSKEY); - int32_t columns = getNumOfColumns(pTableMeta); - SSchema* pSchema = getTableColumnSchema(pTableMeta); - for (int32_t i = 0; i < columns; ++i) { - if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { - result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; - } - } - result += (int32_t)TD_BITMAP_BYTES(columns - 1); - return result; -} -#endif void insDestroyBlockArrayList(SArray* pDataBlockList) { if (pDataBlockList == NULL) { @@ -357,51 +338,6 @@ void insDestroyBlockHashmap(SHashObj* pDataBlockHash) { taosHashCleanup(pDataBlockHash); } -#if 0 -// data block is disordered, sort it in ascending order -void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - - // size is less than the total size, since duplicated rows may be removed yet. - assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size); - - if (!dataBuf->ordered) { - char* pBlockData = pBlocks->data; - - // todo. qsort is unstable, if timestamp is same, should get the last one - taosSort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); - - int32_t i = 0; - int32_t j = 1; - - // delete rows with timestamp conflicts - while (j < pBlocks->numOfRows) { - TSKEY ti = *(TSKEY*)(pBlockData + dataBuf->rowSize * i); - TSKEY tj = *(TSKEY*)(pBlockData + dataBuf->rowSize * j); - - if (ti == tj) { - ++j; - continue; - } - - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); - } - - ++j; - } - - dataBuf->ordered = true; - - pBlocks->numOfRows = i + 1; - dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; - } - - dataBuf->prevTS = INT64_MIN; -} -#endif - // data block is disordered, sort it in ascending order static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; @@ -994,24 +930,24 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { } } -int32_t insBuildOutput(SInsertParseContext* pCxt) { - size_t numOfVg = taosArrayGetSize(pCxt->pVgDataBlocks); - pCxt->pOutput->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - if (NULL == pCxt->pOutput->pDataBlocks) { +int32_t insBuildOutput(SVnodeModifOpStmt* pStmt) { + size_t numOfVg = taosArrayGetSize(pStmt->pVgDataBlocks); + pStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pStmt->pDataBlocks) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } for (size_t i = 0; i < numOfVg; ++i) { - STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); + STableDataBlocks* src = taosArrayGetP(pStmt->pVgDataBlocks, i); SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); + taosHashGetDup(pStmt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; TSWAP(dst->pData, src->pData); buildMsgHeader(src, dst); - taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); + taosArrayPush(pStmt->pDataBlocks, &dst); } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d98d513d5d..466c6edd24 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -612,62 +612,7 @@ static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) { return TSDB_CODE_SUCCESS; } -static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCache* pMetaCache, - SCatalogReq* pCatalogReq) { - int32_t ndbs = taosHashGetSize(pMetaCache->pInsertTables); - pCatalogReq->pTableMeta = taosArrayInit(ndbs, sizeof(STablesReq)); - if (NULL == pCatalogReq->pTableMeta) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pCatalogReq->pTableHash = taosArrayInit(ndbs, sizeof(STablesReq)); - if (NULL == pCatalogReq->pTableHash) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pCatalogReq->pUser = taosArrayInit(ndbs, sizeof(SUserAuthInfo)); - if (NULL == pCatalogReq->pUser) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pCxt->pTableMetaPos = taosArrayInit(pMetaCache->sqlTableNum, sizeof(int32_t)); - pCxt->pTableVgroupPos = taosArrayInit(pMetaCache->sqlTableNum, sizeof(int32_t)); - - int32_t metaReqNo = 0; - int32_t vgroupReqNo = 0; - SInsertTablesMetaReq* p = taosHashIterate(pMetaCache->pInsertTables, NULL); - while (NULL != p) { - STablesReq req = {0}; - strcpy(req.dbFName, p->dbFName); - TSWAP(req.pTables, p->pTableMetaReq); - taosArrayPush(pCatalogReq->pTableMeta, &req); - - req.pTables = NULL; - TSWAP(req.pTables, p->pTableVgroupReq); - taosArrayPush(pCatalogReq->pTableHash, &req); - - int32_t ntables = taosArrayGetSize(p->pTableMetaPos); - for (int32_t i = 0; i < ntables; ++i) { - taosArrayInsert(pCxt->pTableMetaPos, *(int32_t*)taosArrayGet(p->pTableMetaPos, i), &metaReqNo); - ++metaReqNo; - } - - ntables = taosArrayGetSize(p->pTableVgroupPos); - for (int32_t i = 0; i < ntables; ++i) { - taosArrayInsert(pCxt->pTableVgroupPos, *(int32_t*)taosArrayGet(p->pTableVgroupPos, i), &vgroupReqNo); - ++vgroupReqNo; - } - - SUserAuthInfo auth = {0}; - snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser); - snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName); - auth.type = AUTH_TYPE_WRITE; - taosArrayPush(pCatalogReq->pUser, &auth); - - p = taosHashIterate(pMetaCache->pInsertTables, p); - } - return TSDB_CODE_SUCCESS; -} - -int32_t buildCatalogReqForQuery(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { +int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { int32_t code = buildTableReqFromDb(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = buildDbReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup); @@ -697,13 +642,6 @@ int32_t buildCatalogReqForQuery(const SParseMetaCache* pMetaCache, SCatalogReq* return code; } -int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { - if (NULL != pMetaCache->pInsertTables) { - return buildCatalogReqForInsert(pCxt, pMetaCache, pCatalogReq); - } - return buildCatalogReqForQuery(pMetaCache, pCatalogReq); -} - static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj** pHash) { if (NULL == *pHash) { *pHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -791,8 +729,7 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas return TSDB_CODE_SUCCESS; } -int32_t putMetaDataToCacheForQuery(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, - SParseMetaCache* pMetaCache) { +int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { int32_t code = putDbTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = putDbDataToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, &pMetaCache->pDbVgroup); @@ -822,30 +759,6 @@ int32_t putMetaDataToCacheForQuery(const SCatalogReq* pCatalogReq, const SMetaDa return code; } -int32_t putMetaDataToCacheForInsert(const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { - int32_t ndbs = taosArrayGetSize(pMetaData->pUser); - for (int32_t i = 0; i < ndbs; ++i) { - SMetaRes* pRes = taosArrayGet(pMetaData->pUser, i); - if (TSDB_CODE_SUCCESS != pRes->code) { - return pRes->code; - } - if (!(*(bool*)pRes->pRes)) { - return TSDB_CODE_PAR_PERMISSION_DENIED; - } - } - pMetaCache->pTableMetaData = pMetaData->pTableMeta; - pMetaCache->pTableVgroupData = pMetaData->pTableHash; - return TSDB_CODE_SUCCESS; -} - -int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, - bool insertValuesStmt) { - if (insertValuesStmt) { - return putMetaDataToCacheForInsert(pMetaData, pMetaCache); - } - return putMetaDataToCacheForQuery(pCatalogReq, pMetaData, pMetaCache); -} - static int32_t reserveTableReqInCacheImpl(const char* pTbFName, int32_t len, SHashObj** pTables) { if (NULL == *pTables) { *pTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); @@ -1146,82 +1059,6 @@ int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes) { return TSDB_CODE_SUCCESS; } -static int32_t reserveTableReqInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo, - SInsertTablesMetaReq* pReq) { - switch (reqType) { - case CATALOG_REQ_TYPE_META: - taosArrayPush(pReq->pTableMetaReq, pName); - taosArrayPush(pReq->pTableMetaPos, &tableNo); - break; - case CATALOG_REQ_TYPE_VGROUP: - taosArrayPush(pReq->pTableVgroupReq, pName); - taosArrayPush(pReq->pTableVgroupPos, &tableNo); - break; - case CATALOG_REQ_TYPE_BOTH: - taosArrayPush(pReq->pTableMetaReq, pName); - taosArrayPush(pReq->pTableMetaPos, &tableNo); - taosArrayPush(pReq->pTableVgroupReq, pName); - taosArrayPush(pReq->pTableVgroupPos, &tableNo); - break; - default: - break; - } - return TSDB_CODE_SUCCESS; -} - -static int32_t reserveTableReqInDbCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo, - SHashObj* pDbs) { - SInsertTablesMetaReq req = {.pTableMetaReq = taosArrayInit(4, sizeof(SName)), - .pTableMetaPos = taosArrayInit(4, sizeof(int32_t)), - .pTableVgroupReq = taosArrayInit(4, sizeof(SName)), - .pTableVgroupPos = taosArrayInit(4, sizeof(int32_t))}; - tNameGetFullDbName(pName, req.dbFName); - int32_t code = reserveTableReqInCacheForInsert(pName, reqType, tableNo, &req); - if (TSDB_CODE_SUCCESS == code) { - code = taosHashPut(pDbs, pName->dbname, strlen(pName->dbname), &req, sizeof(SInsertTablesMetaReq)); - } - return code; -} - -int32_t reserveTableMetaInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo, - SParseMetaCache* pMetaCache) { - if (NULL == pMetaCache->pInsertTables) { - pMetaCache->pInsertTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (NULL == pMetaCache->pInsertTables) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - pMetaCache->sqlTableNum = tableNo; - SInsertTablesMetaReq* pReq = taosHashGet(pMetaCache->pInsertTables, pName->dbname, strlen(pName->dbname)); - if (NULL == pReq) { - return reserveTableReqInDbCacheForInsert(pName, reqType, tableNo, pMetaCache->pInsertTables); - } - return reserveTableReqInCacheForInsert(pName, reqType, tableNo, pReq); -} - -int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* pMetaCache, int32_t tableNo, - STableMeta** pMeta) { - int32_t reqIndex = *(int32_t*)taosArrayGet(pTableMetaPos, tableNo); - SMetaRes* pRes = taosArrayGet(pMetaCache->pTableMetaData, reqIndex); - if (TSDB_CODE_SUCCESS == pRes->code) { - *pMeta = tableMetaDup((const STableMeta*)pRes->pRes); - if (NULL == *pMeta) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - return pRes->code; -} - -int32_t getTableVgroupFromCacheForInsert(SArray* pTableVgroupPos, SParseMetaCache* pMetaCache, int32_t tableNo, - SVgroupInfo* pVgroup) { - int32_t reqIndex = *(int32_t*)taosArrayGet(pTableVgroupPos, tableNo); - SMetaRes* pRes = taosArrayGet(pMetaCache->pTableVgroupData, reqIndex); - if (TSDB_CODE_SUCCESS == pRes->code) { - memcpy(pVgroup, pRes->pRes, sizeof(SVgroupInfo)); - } - return pRes->code; -} - void destoryParseTablesMetaReqHash(SHashObj* pHash) { SParseTablesMetaReq* p = taosHashIterate(pHash, NULL); while (NULL != p) { @@ -1239,16 +1076,6 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { taosHashCleanup(pMetaCache->pTableMeta); taosHashCleanup(pMetaCache->pTableVgroup); } - SInsertTablesMetaReq* p = taosHashIterate(pMetaCache->pInsertTables, NULL); - while (NULL != p) { - taosArrayDestroy(p->pTableMetaPos); - taosArrayDestroy(p->pTableMetaReq); - taosArrayDestroy(p->pTableVgroupPos); - taosArrayDestroy(p->pTableVgroupReq); - - p = taosHashIterate(pMetaCache->pInsertTables, p); - } - taosHashCleanup(pMetaCache->pInsertTables); taosHashCleanup(pMetaCache->pDbVgroup); taosHashCleanup(pMetaCache->pDbCfg); taosHashCleanup(pMetaCache->pDbInfo); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 748478778a..cf338d63ff 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -167,7 +167,7 @@ static void rewriteExprAlias(SNode* pRoot) { int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { int32_t code = TSDB_CODE_SUCCESS; if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { - code = parseInsertSql(pCxt, pQuery, NULL); + code = parseInsertSql(pCxt, pQuery, NULL, NULL); } else { code = parseSqlIntoAst(pCxt, pQuery); } @@ -175,21 +175,26 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { return code; } -int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { +static int32_t parseQuerySyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { SParseMetaCache metaCache = {0}; - int32_t code = nodesAcquireAllocator(pCxt->allocatorId); + int32_t code = parseSqlSyntax(pCxt, pQuery, &metaCache); + if (TSDB_CODE_SUCCESS == code) { + code = buildCatalogReq(&metaCache, pCatalogReq); + } + destoryParseMetaCache(&metaCache, true); + return code; +} + +int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { + int32_t code = nodesAcquireAllocator(pCxt->allocatorId); if (TSDB_CODE_SUCCESS == code) { if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { - code = parseInsertSyntax(pCxt, pQuery, &metaCache); + code = parseInsertSql(pCxt, pQuery, pCatalogReq, NULL); } else { - code = parseSqlSyntax(pCxt, pQuery, &metaCache); + code = parseQuerySyntax(pCxt, pQuery, pCatalogReq); } } - if (TSDB_CODE_SUCCESS == code) { - code = buildCatalogReq(pCxt, &metaCache, pCatalogReq); - } nodesReleaseAllocator(pCxt->allocatorId); - destoryParseMetaCache(&metaCache, true); terrno = code; return code; } @@ -199,14 +204,10 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata SParseMetaCache metaCache = {0}; int32_t code = nodesAcquireAllocator(pCxt->allocatorId); if (TSDB_CODE_SUCCESS == code) { - code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot); + code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache); } if (TSDB_CODE_SUCCESS == code) { - if (NULL == pQuery->pRoot) { - code = parseInsertSql(pCxt, &pQuery, &metaCache); - } else { - code = analyseSemantic(pCxt, pQuery, &metaCache); - } + code = analyseSemantic(pCxt, pQuery, &metaCache); } nodesReleaseAllocator(pCxt->allocatorId); destoryParseMetaCache(&metaCache, false); @@ -214,6 +215,11 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata return code; } +int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, + SQuery* pQuery) { + return parseInsertSql(pCxt, &pQuery, pCatalogReq, pMetaData); +} + void qDestroyParseContext(SParseContext* pCxt) { if (NULL == pCxt) { return; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 7f18a7b282..96ba8ab273 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -228,11 +228,23 @@ int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, SRequestConnInfo* pConn return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta); } +int32_t __catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + STableMeta** pTableMeta) { + return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta, true); +} + int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* vgInfo) { return g_mockCatalogService->catalogGetTableHashVgroup(pTableName, vgInfo); } +int32_t __catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + SVgroupInfo* pVgroup, bool* exists) { + int32_t code = g_mockCatalogService->catalogGetTableHashVgroup(pTableName, pVgroup, true); + *exists = 0 != pVgroup->vgId; + return code; +} + int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pVgList) { return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList); @@ -289,8 +301,11 @@ void initMetaDataEnv() { static Stub stub; stub.set(catalogGetHandle, __catalogGetHandle); stub.set(catalogGetTableMeta, __catalogGetTableMeta); + stub.set(catalogGetCachedTableMeta, __catalogGetCachedTableMeta); stub.set(catalogGetSTableMeta, __catalogGetTableMeta); + stub.set(catalogGetCachedSTableMeta, __catalogGetCachedTableMeta); stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); + stub.set(catalogGetCachedTableHashVgroup, __catalogGetCachedTableHashVgroup); stub.set(catalogGetTableDistVgInfo, __catalogGetTableDistVgInfo); stub.set(catalogGetDBVgVersion, __catalogGetDBVgVersion); stub.set(catalogGetDBVgList, __catalogGetDBVgList); diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index f2cebcb08d..95f7af435d 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -91,7 +91,7 @@ class MockCatalogServiceImpl { public: static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); - MockCatalogServiceImpl() : id_(1) {} + MockCatalogServiceImpl() : id_(1), havaCache_(true) {} ~MockCatalogServiceImpl() { for (auto& cfg : dbCfg_) { @@ -106,7 +106,11 @@ class MockCatalogServiceImpl { int32_t catalogGetHandle() const { return 0; } - int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { + int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, bool onlyCache = false) const { + if (onlyCache && !havaCache_) { + return TSDB_CODE_SUCCESS; + } + std::unique_ptr table; char db[TSDB_DB_NAME_LEN] = {0}; @@ -121,7 +125,12 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { + int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, bool onlyCache = false) const { + if (onlyCache && !havaCache_) { + vgInfo->vgId = 0; + return TSDB_CODE_SUCCESS; + } + vgInfo->vgId = 1; return TSDB_CODE_SUCCESS; } @@ -618,6 +627,7 @@ class MockCatalogServiceImpl { IndexMetaCache index_; DnodeCache dnode_; DbCfgCache dbCfg_; + bool havaCache_; }; MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {} @@ -651,12 +661,14 @@ void MockCatalogService::createDatabase(const std::string& db, bool rollup, int8 impl_->createDatabase(db, rollup, cacheLast); } -int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { - return impl_->catalogGetTableMeta(pTableName, pTableMeta); +int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, + bool onlyCache) const { + return impl_->catalogGetTableMeta(pTableName, pTableMeta, onlyCache); } -int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { - return impl_->catalogGetTableHashVgroup(pTableName, vgInfo); +int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, + bool onlyCache) const { + return impl_->catalogGetTableHashVgroup(pTableName, vgInfo, onlyCache); } int32_t MockCatalogService::catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const { diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index d9d2185728..acd7fab8e1 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -67,8 +67,8 @@ class MockCatalogService { void createDnode(int32_t dnodeId, const std::string& host, int16_t port); void createDatabase(const std::string& db, bool rollup = false, int8_t cacheLast = 0); - int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const; - int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const; + int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, bool onlyCache = false) const; + int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, bool onlyCache = false) const; int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const; int32_t catalogGetDBVgList(const char* pDbFName, SArray** pVgList) const; int32_t catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pDbCfg) const; diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index bf27fd2e13..1063f4fadc 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -233,16 +233,15 @@ class ParserTestBaseImpl { } void doBuildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { - DO_WITH_THROW(buildCatalogReq, pCxt, pMetaCache, pCatalogReq); + DO_WITH_THROW(buildCatalogReq, pMetaCache, pCatalogReq); } void doGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) { DO_WITH_THROW(g_mockCatalogService->catalogGetAllMeta, pCatalogReq, pMetaData); } - void doPutMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, - bool isInsertValues) { - DO_WITH_THROW(putMetaDataToCache, pCatalogReq, pMetaData, pMetaCache, isInsertValues); + void doPutMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { + DO_WITH_THROW(putMetaDataToCache, pCatalogReq, pMetaData, pMetaCache); } void doAuthenticate(SParseContext* pCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) { @@ -280,15 +279,14 @@ class ParserTestBaseImpl { res_.calcConstAst_ = toString(pQuery->pRoot); } - void doParseInsertSql(SParseContext* pCxt, SQuery** pQuery, SParseMetaCache* pMetaCache) { - DO_WITH_THROW(parseInsertSql, pCxt, pQuery, pMetaCache); + void doParseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData) { + DO_WITH_THROW(parseInsertSql, pCxt, pQuery, pCatalogReq, pMetaData); ASSERT_NE(*pQuery, nullptr); res_.parsedAst_ = toString((*pQuery)->pRoot); } - void doParseInsertSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCache* pMetaCache) { - DO_WITH_THROW(parseInsertSyntax, pCxt, pQuery, pMetaCache); - ASSERT_NE(*pQuery, nullptr); + void doContinueParseSql(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SQuery* pQuery) { + DO_WITH_THROW(qContinueParseSql, pCxt, pCatalogReq, pMetaData, pQuery); } string toString(const SNode* pRoot) { @@ -310,7 +308,7 @@ class ParserTestBaseImpl { if (qIsInsertValuesSql(cxt.pSql, cxt.sqlLen)) { unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); - doParseInsertSql(&cxt, query.get(), nullptr); + doParseInsertSql(&cxt, query.get(), nullptr, nullptr); } else { unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParse(&cxt, query.get()); @@ -356,61 +354,102 @@ class ParserTestBaseImpl { } } - void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) { - reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL); - try { - unique_ptr > cxt(new SParseContext(), destoryParseContext); - setParseContext(sql, cxt.get(), true); - - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); - bool request = true; - unique_ptr > metaCache( - new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request))); - bool isInsertValues = qIsInsertValuesSql(cxt->pSql, cxt->sqlLen); - if (isInsertValues) { - doParseInsertSyntax(cxt.get(), query.get(), metaCache.get()); - } else { - doParse(cxt.get(), query.get()); - doCollectMetaKey(cxt.get(), *(query.get()), metaCache.get()); + void runQueryAsyncInternalFuncs(SParseContext* pParCxt) { + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); + bool request = true; + unique_ptr > metaCache( + new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request))); + doParse(pParCxt, query.get()); + doCollectMetaKey(pParCxt, *(query.get()), metaCache.get()); + + SQuery* pQuery = *(query.get()); + + unique_ptr catalogReq(new SCatalogReq(), + MockCatalogService::destoryCatalogReq); + doBuildCatalogReq(pParCxt, metaCache.get(), catalogReq.get()); + + string err; + thread t1([&]() { + try { + unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); + doGetAllMeta(catalogReq.get(), metaData.get()); + + metaCache.reset(new SParseMetaCache()); + request = false; + doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get()); + + doAuthenticate(pParCxt, pQuery, metaCache.get()); + + doTranslate(pParCxt, pQuery, metaCache.get()); + + doCalculateConstant(pParCxt, pQuery); + } catch (const TerminateFlag& e) { + // success and terminate + } catch (const runtime_error& e) { + err = e.what(); + } catch (...) { + err = "unknown error"; } + }); - SQuery* pQuery = *(query.get()); - - unique_ptr catalogReq(new SCatalogReq(), - MockCatalogService::destoryCatalogReq); - doBuildCatalogReq(cxt.get(), metaCache.get(), catalogReq.get()); + t1.join(); + if (!err.empty()) { + throw runtime_error(err); + } + } - string err; - thread t1([&]() { - try { - unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); - doGetAllMeta(catalogReq.get(), metaData.get()); + void runInsertAsyncInternalFuncsImpl(SParseContext* pParCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, + SMetaData* pMetaData) { + doParseInsertSql(pParCxt, pQuery, pCatalogReq, pMetaData); - metaCache.reset(new SParseMetaCache()); - request = false; - doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get(), isInsertValues); + if (QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { + return; + } - if (isInsertValues) { - doParseInsertSql(cxt.get(), query.get(), metaCache.get()); - } else { - doAuthenticate(cxt.get(), pQuery, metaCache.get()); + string err; + thread t1([&]() { + try { + doGetAllMeta(pCatalogReq, pMetaData); - doTranslate(cxt.get(), pQuery, metaCache.get()); + doParseInsertSql(pParCxt, pQuery, pCatalogReq, pMetaData); - doCalculateConstant(cxt.get(), pQuery); - } - } catch (const TerminateFlag& e) { - // success and terminate - } catch (const runtime_error& e) { - err = e.what(); - } catch (...) { - err = "unknown error"; + if (QUERY_EXEC_STAGE_SCHEDULE != (*pQuery)->execStage) { + runInsertAsyncInternalFuncsImpl(pParCxt, pQuery, pCatalogReq, pMetaData); } - }); + } catch (const TerminateFlag& e) { + // success and terminate + } catch (const runtime_error& e) { + err = e.what(); + } catch (...) { + err = "unknown error"; + } + }); + + t1.join(); + if (!err.empty()) { + throw runtime_error(err); + } + } + + void runInsertAsyncInternalFuncs(SParseContext* pParCxt) { + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); + unique_ptr catalogReq(new SCatalogReq(), + MockCatalogService::destoryCatalogReq); + unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); + runInsertAsyncInternalFuncsImpl(pParCxt, query.get(), catalogReq.get(), metaData.get()); + } + + void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) { + reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL); + try { + unique_ptr > cxt(new SParseContext(), destoryParseContext); + setParseContext(sql, cxt.get(), true); - t1.join(); - if (!err.empty()) { - throw runtime_error(err); + bool isInsertValues = qIsInsertValuesSql(cxt->pSql, cxt->sqlLen); + if (isInsertValues) { + runInsertAsyncInternalFuncs(cxt.get()); + } else { + runQueryAsyncInternalFuncs(cxt.get()); } if (g_dump) { @@ -437,25 +476,39 @@ class ParserTestBaseImpl { doParseSqlSyntax(cxt.get(), query.get(), catalogReq.get()); SQuery* pQuery = *(query.get()); - string err; - thread t1([&]() { - try { - unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); - doGetAllMeta(catalogReq.get(), metaData.get()); - - doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery); - } catch (const TerminateFlag& e) { - // success and terminate - } catch (const runtime_error& e) { - err = e.what(); - } catch (...) { - err = "unknown error"; + switch (pQuery->execStage) { + case QUERY_EXEC_STAGE_PARSE: + case QUERY_EXEC_STAGE_ANALYSE: { + string err; + thread t1([&]() { + try { + unique_ptr metaData(new SMetaData(), + MockCatalogService::destoryMetaData); + doGetAllMeta(catalogReq.get(), metaData.get()); + if (QUERY_EXEC_STAGE_PARSE == pQuery->execStage) { + doContinueParseSql(cxt.get(), catalogReq.get(), metaData.get(), pQuery); + } else { + doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery); + } + } catch (const TerminateFlag& e) { + // success and terminate + } catch (const runtime_error& e) { + err = e.what(); + } catch (...) { + err = "unknown error"; + } + }); + + t1.join(); + if (!err.empty()) { + throw runtime_error(err); + } + break; } - }); - - t1.join(); - if (!err.empty()) { - throw runtime_error(err); + case QUERY_EXEC_STAGE_SCHEDULE: + break; + default: + break; } if (g_dump) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index c26ae4e646..e07255b66b 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1134,7 +1134,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } -#if 1 +#if 0 SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; -- GitLab