Skip to content

TDengine
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
TDengine
TDengine
提交 8513dc10 编写于 作者: X Xiaoyu Wang
浏览文件
操作

enh: insert optimize

上级 bb490f07
隐藏空白更改
内联 并排
Showing with 1640 addition and 1572 deletion
include/libs/catalog/catalog.h
浏览文件 @ 8513dc10
......@@ -186,6 +186,8 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId,
*/
int32_t catalogGetTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta);
int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta);
/**
* Get a super table's meta data.
......@@ -198,6 +200,8 @@ int32_t catalogGetTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const S
*/
int32_t catalogGetSTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta);
int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta);
int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
......@@ -261,7 +265,8 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, SRequestConnInfo* pConn, c
* @return error code
*/
int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pName, SVgroupInfo* vgInfo);
int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
SVgroupInfo* pVgroup, bool* exists);
/**
* Get all meta data required in pReq.
* @param pCatalog (input, got with catalogGetHandle)
......
include/libs/nodes/querynodes.h
浏览文件 @ 8513dc10
......@@ -354,12 +354,33 @@ typedef struct SVgDataBlocks {
void* pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ...
} SVgDataBlocks;
typedef void (*FFreeDataBlockHash)(SHashObj*);
typedef void (*FFreeDataBlockArray)(SArray*);
typedef struct SVnodeModifOpStmt {
ENodeType nodeType;
ENodeType sqlNodeType;
SArray* pDataBlocks; // data block for each vgroup, SArray<SVgDataBlocks*>.
uint32_t insertType; // insert data from [file|sql statement| bound statement]
const char* sql; // current sql statement position
ENodeType nodeType;
ENodeType sqlNodeType;
SArray* pDataBlocks; // data block for each vgroup, SArray<SVgDataBlocks*>.
uint32_t insertType; // insert data from [file|sql statement| bound statement]
const char* pSql; // current sql statement position
int32_t totalRowsNum;
int32_t totalTbNum;
SName targetTableName;
SName usingTableName;
const char* pBoundCols;
struct STableMeta* pTableMeta;
SHashObj* pVgroupsHashObj;
SHashObj* pTableBlockHashObj;
SHashObj* pSubTableHashObj;
SHashObj* pTableNameHashObj;
SHashObj* pDbFNameHashObj;
SArray* pVgDataBlocks;
SVCreateTbReq createTblReq;
TdFilePtr fp;
FFreeDataBlockHash freeHashFunc;
FFreeDataBlockArray freeArrayFunc;
bool usingTableProcessing;
bool fileProcessing;
} SVnodeModifOpStmt;
typedef struct SExplainOptions {
......@@ -389,24 +410,32 @@ typedef enum EQueryExecMode {
QUERY_EXEC_MODE_EMPTY_RESULT
} EQueryExecMode;
typedef enum EQueryExecStage {
QUERY_EXEC_STAGE_PARSE = 1,
QUERY_EXEC_STAGE_ANALYSE,
QUERY_EXEC_STAGE_SCHEDULE,
QUERY_EXEC_STAGE_END
} EQueryExecStage;
typedef struct SQuery {
ENodeType type;
EQueryExecMode execMode;
bool haveResultSet;
SNode* pRoot;
int32_t numOfResCols;
SSchema* pResSchema;
int8_t precision;
SCmdMsgInfo* pCmdMsg;
int32_t msgType;
SArray* pTargetTableList;
SArray* pTableList;
SArray* pDbList;
bool showRewrite;
int32_t placeholderNum;
SArray* pPlaceholderValues;
SNode* pPrepareRoot;
bool stableQuery;
ENodeType type;
EQueryExecStage execStage;
EQueryExecMode execMode;
bool haveResultSet;
SNode* pRoot;
int32_t numOfResCols;
SSchema* pResSchema;
int8_t precision;
SCmdMsgInfo* pCmdMsg;
int32_t msgType;
SArray* pTargetTableList;
SArray* pTableList;
SArray* pDbList;
bool showRewrite;
int32_t placeholderNum;
SArray* pPlaceholderValues;
SNode* pPrepareRoot;
bool stableQuery;
} SQuery;
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
......
include/libs/parser/parser.h
浏览文件 @ 8513dc10
......@@ -64,8 +64,6 @@ typedef struct SParseContext {
SArray* pTableMetaPos; // sql table pos => catalog data pos
SArray* pTableVgroupPos; // sql table pos => catalog data pos
int64_t allocatorId;
bool needMultiParse;
SParseCsvCxt csvCxt;
} SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
......@@ -75,6 +73,8 @@ bool qIsInsertValuesSql(const char* pStr, size_t length);
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq);
int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq,
const struct SMetaData* pMetaData, SQuery* pQuery);
int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData,
SQuery* pQuery);
void qDestroyParseContext(SParseContext* pCxt);
......
source/client/inc/clientInt.h
浏览文件 @ 8513dc10
......@@ -379,7 +379,6 @@ void hbMgrInitMqHbRspHandle();
typedef struct SSqlCallbackWrapper {
SParseContext* pParseCtx;
SCatalogReq* pCatalogReq;
SMetaData* pResultMeta;
SRequestObj* pRequest;
} SSqlCallbackWrapper;
......@@ -393,7 +392,7 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList);
int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);
int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog);
bool qnodeRequired(SRequestObj* pRequest);
int32_t continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest);
void continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest);
void destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper);
#ifdef __cplusplus
......
source/client/src/clientImpl.c
浏览文件 @ 8513dc10
......@@ -867,6 +867,10 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
return code;
}
static bool incompletaFileParsing(SNode* pStmt) {
return QUERY_NODE_VNODE_MODIF_STMT != nodeType(pStmt) ? false : ((SVnodeModifOpStmt*)pStmt)->fileProcessing;
}
// todo refacto the error code mgmt
void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
SSqlCallbackWrapper* pWrapper = param;
......@@ -921,11 +925,9 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
pRequest->code = code1;
}
if (pRequest->code == TSDB_CODE_SUCCESS && NULL != pWrapper->pParseCtx && pWrapper->pParseCtx->needMultiParse) {
code = continueInsertFromCsv(pWrapper, pRequest);
if (TSDB_CODE_SUCCESS == code) {
return;
}
if (pRequest->code == TSDB_CODE_SUCCESS && incompletaFileParsing(pRequest->pQuery->pRoot)) {
continueInsertFromCsv(pWrapper, pRequest);
return;
}
destorySqlCallbackWrapper(pWrapper);
......@@ -1049,7 +1051,9 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
}
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
SArray* pNodeList = NULL;
buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
if (QUERY_NODE_VNODE_MODIF_STMT != nodeType(pRequest->pQuery->pRoot)) {
buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
}
SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter,
.requestId = pRequest->requestId,
......
source/client/src/clientMain.c
浏览文件 @ 8513dc10
......@@ -682,11 +682,10 @@ void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) {
}
destoryCatalogReq(pWrapper->pCatalogReq);
qDestroyParseContext(pWrapper->pParseCtx);
catalogFreeMetaData(pWrapper->pResultMeta);
taosMemoryFree(pWrapper);
}
void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t code) {
SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param;
SRequestObj *pRequest = pWrapper->pRequest;
SQuery *pQuery = pRequest->pQuery;
......@@ -704,13 +703,6 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
pRequest->metric.semanticEnd = taosGetTimestampUs();
if (code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) {
pWrapper->pResultMeta = catalogCloneMetaData(pResultMeta);
if (NULL == pWrapper->pResultMeta) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (code == TSDB_CODE_SUCCESS) {
if (pQuery->haveResultSet) {
setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols);
......@@ -747,14 +739,83 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
}
}
int32_t continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) {
qDestroyQuery(pRequest->pQuery);
pRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pRequest->pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
static int32_t getAllMetaAsync(SSqlCallbackWrapper *pWrapper, catalogCallback fp) {
SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter,
.requestId = pWrapper->pParseCtx->requestId,
.requestObjRefId = pWrapper->pParseCtx->requestRid,
.mgmtEps = pWrapper->pParseCtx->mgmtEpSet};
pWrapper->pRequest->metric.ctgStart = taosGetTimestampUs();
return catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, fp, pWrapper,
&pWrapper->pRequest->body.queryJob);
}
static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t code);
static int32_t phaseAsyncQuery(SSqlCallbackWrapper *pWrapper) {
int32_t code = TSDB_CODE_SUCCESS;
switch (pWrapper->pRequest->pQuery->execStage) {
case QUERY_EXEC_STAGE_PARSE: {
// continue parse after get metadata
code = getAllMetaAsync(pWrapper, doAsyncQueryFromParse);
break;
}
case QUERY_EXEC_STAGE_ANALYSE: {
// analysis after get metadata
code = getAllMetaAsync(pWrapper, doAsyncQueryFromAnalyse);
break;
}
case QUERY_EXEC_STAGE_SCHEDULE: {
launchAsyncQuery(pWrapper->pRequest, pWrapper->pRequest->pQuery, NULL, pWrapper);
break;
}
default:
break;
}
return code;
}
static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t code) {
SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param;
SRequestObj *pRequest = pWrapper->pRequest;
SQuery *pQuery = pRequest->pQuery;
pRequest->metric.ctgEnd = taosGetTimestampUs();
qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId);
if (code == TSDB_CODE_SUCCESS) {
code = qContinueParseSql(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery);
}
if (TSDB_CODE_SUCCESS == code) {
code = phaseAsyncQuery(pWrapper);
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code,
tstrerror(code), pWrapper->pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
terrno = code;
pWrapper->pRequest->code = code;
pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code);
}
}
void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) {
int32_t code = qParseSqlSyntax(pWrapper->pParseCtx, &pRequest->pQuery, pWrapper->pCatalogReq);
if (TSDB_CODE_SUCCESS == code) {
code = phaseAsyncQuery(pWrapper);
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code,
tstrerror(code), pWrapper->pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
terrno = code;
pWrapper->pRequest->code = code;
pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code);
}
retrieveMetaCallback(pWrapper->pResultMeta, pWrapper, TSDB_CODE_SUCCESS);
return TSDB_CODE_SUCCESS;
}
void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
......@@ -836,26 +897,16 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
if (TSDB_CODE_SUCCESS == code && !updateMetaForce) {
SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary;
if (NULL == pRequest->pQuery->pRoot) {
if (QUERY_NODE_INSERT_STMT == nodeType(pRequest->pQuery->pRoot)) {
atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
} else if (QUERY_NODE_SELECT_STMT == pRequest->pQuery->pRoot->type) {
} else if (QUERY_NODE_SELECT_STMT == nodeType(pRequest->pQuery->pRoot)) {
atomic_add_fetch_64((int64_t *)&pActivity->numOfQueryReq, 1);
}
}
if (TSDB_CODE_SUCCESS == code) {
SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter,
.requestId = pWrapper->pParseCtx->requestId,
.requestObjRefId = pWrapper->pParseCtx->requestRid,
.mgmtEps = pWrapper->pParseCtx->mgmtEpSet};
pRequest->metric.ctgStart = taosGetTimestampUs();
code = catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, retrieveMetaCallback,
pWrapper, &pRequest->body.queryJob);
}
if (TSDB_CODE_SUCCESS != code) {
phaseAsyncQuery(pWrapper);
} else {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
......
source/libs/nodes/src/nodesUtilFuncs.c
浏览文件 @ 8513dc10
......@@ -791,9 +791,24 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode((SNode*)pStmt->pSlimit);
break;
}
case QUERY_NODE_VNODE_MODIF_STMT:
destroyVgDataBlockArray(((SVnodeModifOpStmt*)pNode)->pDataBlocks);
case QUERY_NODE_VNODE_MODIF_STMT: {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pNode;
destroyVgDataBlockArray(pStmt->pDataBlocks);
taosMemoryFreeClear(pStmt->pTableMeta);
taosHashCleanup(pStmt->pVgroupsHashObj);
taosHashCleanup(pStmt->pSubTableHashObj);
taosHashCleanup(pStmt->pTableNameHashObj);
taosHashCleanup(pStmt->pDbFNameHashObj);
if (pStmt->freeHashFunc) {
pStmt->freeHashFunc(pStmt->pTableBlockHashObj);
}
if (pStmt->freeArrayFunc) {
pStmt->freeArrayFunc(pStmt->pVgDataBlocks);
}
tdDestroySVCreateTbReq(&pStmt->createTblReq);
taosCloseFile(&pStmt->fp);
break;
}
case QUERY_NODE_CREATE_DATABASE_STMT:
nodesDestroyNode((SNode*)((SCreateDatabaseStmt*)pNode)->pOptions);
break;
......
source/libs/parser/inc/parInsertUtil.h
浏览文件 @ 8513dc10
......@@ -79,29 +79,6 @@ typedef struct SInsertParseBaseContext {
SMsgBuf msg;
} SInsertParseBaseContext;
typedef struct SInsertParseContext {
SParseContext *pComCxt; // input
char *pSql; // input
SMsgBuf msg; // input
STableMeta *pTableMeta; // each table
SParsedDataColInfo tags; // each table
SVCreateTbReq createTblReq; // each table
SHashObj *pVgroupsHashObj; // global
SHashObj *pTableBlockHashObj; // global
SHashObj *pSubTableHashObj; // global
SArray *pVgDataBlocks; // global
SHashObj *pTableNameHashObj; // global
SHashObj *pDbFNameHashObj; // global
int32_t totalNum;
SVnodeModifOpStmt *pOutput;
SStmtCallback *pStmtCb;
SParseMetaCache *pMetaCache;
char sTableName[TSDB_TABLE_NAME_LEN];
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
int64_t memElapsed;
int64_t parRowElapsed;
} SInsertParseContext;
typedef struct SInsertParseSyntaxCxt {
SParseContext *pComCxt;
char *pSql;
......@@ -142,7 +119,7 @@ typedef struct STableDataBlocks {
int32_t insGetExtendedRowSize(STableDataBlocks *pBlock);
void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx);
int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows);
int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows, SMsgBuf *pMsg);
int32_t insSchemaIdxCompar(const void *lhs, const void *rhs);
int32_t insBoundIdxCompar(const void *lhs, const void *rhs);
void insSetBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols);
......@@ -161,7 +138,7 @@ void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag
SArray *tagName, uint8_t tagNum);
int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param);
int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start);
int32_t insBuildOutput(SInsertParseContext *pCxt);
int32_t insBuildOutput(SVnodeModifOpStmt *pStmt);
void insDestroyDataBlock(STableDataBlocks *pDataBlock);
#endif // TDENGINE_PAR_INSERT_UTIL_H
source/libs/parser/inc/parInt.h
浏览文件 @ 8513dc10
......@@ -27,8 +27,7 @@ extern "C" {
#define QUERY_SMA_OPTIMIZE_DISABLE 0
#define QUERY_SMA_OPTIMIZE_ENABLE 1
int32_t parseInsertSyntax(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache);
int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache);
int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData);
int32_t parse(SParseContext* pParseCxt, SQuery** pQuery);
int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache);
int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache);
......
source/libs/parser/inc/parUtil.h
浏览文件 @ 8513dc10
......@@ -60,22 +60,17 @@ typedef struct SInsertTablesMetaReq {
} SInsertTablesMetaReq;
typedef struct SParseMetaCache {
SHashObj* pTableMeta; // key is tbFName, element is STableMeta*
SHashObj* pDbVgroup; // key is dbFName, element is SArray<SVgroupInfo>*
SHashObj* pTableVgroup; // key is tbFName, element is SVgroupInfo*
SHashObj* pDbCfg; // key is tbFName, element is SDbCfgInfo*
SHashObj* pDbInfo; // key is tbFName, element is SDbInfo*
SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass
SHashObj* pUdf; // key is funcName, element is SFuncInfo*
SHashObj* pTableIndex; // key is tbFName, element is SArray<STableIndexInfo>*
SHashObj* pTableCfg; // key is tbFName, element is STableCfg*
SArray* pDnodes; // element is SEpSet
bool dnodeRequired;
SHashObj* pInsertTables; // key is dbName, element is SInsertTablesMetaReq*, for insert
const char* pUser;
const SArray* pTableMetaData; // pRes = STableMeta*
const SArray* pTableVgroupData; // pRes = SVgroupInfo*
int32_t sqlTableNum;
SHashObj* pTableMeta; // key is tbFName, element is STableMeta*
SHashObj* pDbVgroup; // key is dbFName, element is SArray<SVgroupInfo>*
SHashObj* pTableVgroup; // key is tbFName, element is SVgroupInfo*
SHashObj* pDbCfg; // key is tbFName, element is SDbCfgInfo*
SHashObj* pDbInfo; // key is tbFName, element is SDbInfo*
SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass
SHashObj* pUdf; // key is funcName, element is SFuncInfo*
SHashObj* pTableIndex; // key is tbFName, element is SArray<STableIndexInfo>*
SHashObj* pTableCfg; // key is tbFName, element is STableCfg*
SArray* pDnodes; // element is SEpSet
bool dnodeRequired;
} SParseMetaCache;
int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...);
......@@ -93,9 +88,8 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta);
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache,
bool insertValuesStmt);
int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache);
int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache);
int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache);
......@@ -122,12 +116,6 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun
int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes);
int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput);
int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes);
int32_t reserveTableMetaInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo,
SParseMetaCache* pMetaCache);
int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* pMetaCache, int32_t tableNo,
STableMeta** pMeta);
int32_t getTableVgroupFromCacheForInsert(SArray* pTableVgroupPos, SParseMetaCache* pMetaCache, int32_t tableNo,
SVgroupInfo* pVgroup);
void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request);
#ifdef __cplusplus
......
source/libs/parser/src/parAstParser.c
浏览文件 @ 8513dc10
......@@ -84,6 +84,7 @@ abort_parse:
(*pQuery)->pRoot = cxt.pRootNode;
(*pQuery)->placeholderNum = cxt.placeholderNo;
TSWAP((*pQuery)->pPlaceholderValues, cxt.pPlaceholderValues);
(*pQuery)->execStage = QUERY_EXEC_STAGE_ANALYSE;
}
taosArrayDestroy(cxt.pPlaceholderValues);
return cxt.errCode;
......
source/libs/parser/src/parInsertSml.c
浏览文件 @ 8513dc10
......@@ -333,11 +333,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
}
SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData);
if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, rowNum)) {
return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX");
}
return TSDB_CODE_SUCCESS;
return insSetBlockInfo(pBlocks, pDataBlock, rowNum, &pBuf);
}
void* smlInitHandle(SQuery* pQuery) {
......
source/libs/parser/src/parInsertSql.c
浏览文件 @ 8513dc10
......@@ -18,123 +18,246 @@
#include "tglobal.h"
#include "ttime.h"
#define NEXT_TOKEN_WITH_PREV(pSql, sToken) \
do { \
int32_t index = 0; \
sToken = tStrGetToken(pSql, &index, true); \
pSql += index; \
#define NEXT_TOKEN_WITH_PREV(pSql, token) \
do { \
int32_t index = 0; \
token = tStrGetToken(pSql, &index, true); \
pSql += index; \
} while (0)
#define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \
do { \
sToken = tStrGetToken(pSql, &index, false); \
#define NEXT_TOKEN_KEEP_SQL(pSql, token, index) \
do { \
token = tStrGetToken(pSql, &index, false); \
} while (0)
#define NEXT_VALID_TOKEN(pSql, sToken) \
do { \
sToken.n = tGetToken(pSql, &sToken.type); \
sToken.z = pSql; \
pSql += sToken.n; \
} while (TK_NK_SPACE == sToken.type)
#define NEXT_VALID_TOKEN(pSql, token) \
do { \
(token).n = tGetToken(pSql, &(token).type); \
(token).z = (char*)pSql; \
pSql += (token).n; \
} while (TK_NK_SPACE == (token).type)
typedef struct SInsertParseContext {
SParseContext* pComCxt;
SMsgBuf msg;
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
SParsedDataColInfo tags; // for stmt
bool missCache;
} SInsertParseContext;
typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param);
static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
static int32_t skipInsertInto(char** pSql, SMsgBuf* pMsg) {
SToken sToken;
NEXT_TOKEN(*pSql, sToken);
if (TK_INSERT != sToken.type && TK_IMPORT != sToken.type) {
return buildSyntaxErrMsg(pMsg, "keyword INSERT is expected", sToken.z);
static bool isNullStr(SToken* pToken) {
return ((pToken->type == TK_NK_STRING) && (strlen(TSDB_DATA_NULL_STR_L) == pToken->n) &&
(strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
}
static bool isNullValue(int8_t dataType, SToken* pToken) {
return TK_NULL == pToken->type || (!IS_STR_DATA_TYPE(dataType) && isNullStr(pToken));
}
static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) {
errno = 0;
*value = taosStr2Double(pToken->z, endPtr);
// not a valid integer number, return error
if ((*endPtr - pToken->z) != pToken->n) {
return TK_NK_ILLEGAL;
}
return pToken->type;
}
static int32_t skipInsertInto(const char** pSql, SMsgBuf* pMsg) {
SToken token;
NEXT_TOKEN(*pSql, token);
if (TK_INSERT != token.type && TK_IMPORT != token.type) {
return buildSyntaxErrMsg(pMsg, "keyword INSERT is expected", token.z);
}
NEXT_TOKEN(*pSql, sToken);
if (TK_INTO != sToken.type) {
return buildSyntaxErrMsg(pMsg, "keyword INTO is expected", sToken.z);
NEXT_TOKEN(*pSql, token);
if (TK_INTO != token.type) {
return buildSyntaxErrMsg(pMsg, "keyword INTO is expected", token.z);
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass) {
SParseContext* pBasicCtx = pCxt->pComCxt;
if (pBasicCtx->async) {
return getUserAuthFromCache(pCxt->pMetaCache, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass);
static int32_t skipParentheses(SInsertParseContext* pCxt, const char** pSql) {
SToken token;
int32_t expectRightParenthesis = 1;
while (1) {
NEXT_TOKEN(*pSql, token);
if (TK_NK_LP == token.type) {
++expectRightParenthesis;
} else if (TK_NK_RP == token.type && 0 == --expectRightParenthesis) {
break;
}
if (0 == token.n) {
return buildSyntaxErrMsg(&pCxt->msg, ") expected", NULL);
}
}
SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter,
.requestId = pBasicCtx->requestId,
.requestObjRefId = pBasicCtx->requestRid,
.mgmtEps = pBasicCtx->mgmtEpSet};
return TSDB_CODE_SUCCESS;
}
return catalogChkAuth(pBasicCtx->pCatalog, &conn, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass);
static int32_t skipTableOptions(SInsertParseContext* pCxt, const char** pSql) {
do {
int32_t index = 0;
SToken token;
NEXT_TOKEN_KEEP_SQL(*pSql, token, index);
if (TK_TTL == token.type || TK_COMMENT == token.type) {
*pSql += index;
NEXT_TOKEN_WITH_PREV(*pSql, token);
} else {
break;
}
} while (1);
return TSDB_CODE_SUCCESS;
}
static int32_t getTableSchema(SInsertParseContext* pCxt, int32_t tbNo, SName* pTbName, bool isStb,
STableMeta** pTableMeta) {
SParseContext* pBasicCtx = pCxt->pComCxt;
if (pBasicCtx->async) {
return getTableMetaFromCacheForInsert(pBasicCtx->pTableMetaPos, pCxt->pMetaCache, tbNo, pTableMeta);
// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
static int32_t ignoreUsingClause(SInsertParseContext* pCxt, const char** pSql) {
int32_t code = TSDB_CODE_SUCCESS;
SToken token;
NEXT_TOKEN(*pSql, token);
NEXT_TOKEN(*pSql, token);
if (TK_NK_LP == token.type) {
code = skipParentheses(pCxt, pSql);
if (TSDB_CODE_SUCCESS == code) {
NEXT_TOKEN(*pSql, token);
}
}
SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter,
.requestId = pBasicCtx->requestId,
.requestObjRefId = pBasicCtx->requestRid,
.mgmtEps = pBasicCtx->mgmtEpSet};
if (isStb) {
return catalogGetSTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta);
// pSql -> TAGS (tag1_value, ...)
if (TSDB_CODE_SUCCESS == code) {
if (TK_TAGS != token.type) {
code = buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", token.z);
} else {
NEXT_TOKEN(*pSql, token);
}
}
if (TSDB_CODE_SUCCESS == code) {
if (TK_NK_LP != token.type) {
code = buildSyntaxErrMsg(&pCxt->msg, "( is expected", token.z);
} else {
code = skipParentheses(pCxt, pSql);
}
}
return catalogGetTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta);
}
static int32_t getTableVgroup(SInsertParseContext* pCxt, int32_t tbNo, SName* pTbName, SVgroupInfo* pVg) {
SParseContext* pBasicCtx = pCxt->pComCxt;
if (pBasicCtx->async) {
return getTableVgroupFromCacheForInsert(pBasicCtx->pTableVgroupPos, pCxt->pMetaCache, tbNo, pVg);
if (TSDB_CODE_SUCCESS == code) {
code = skipTableOptions(pCxt, pSql);
}
SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter,
.requestId = pBasicCtx->requestId,
.requestObjRefId = pBasicCtx->requestRid,
.mgmtEps = pBasicCtx->mgmtEpSet};
return catalogGetTableHashVgroup(pBasicCtx->pCatalog, &conn, pTbName, pVg);
return code;
}
static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, bool isStb) {
CHECK_CODE(getTableSchema(pCxt, tbNo, name, isStb, &pCxt->pTableMeta));
if (!isStb) {
SVgroupInfo vg;
CHECK_CODE(getTableVgroup(pCxt, tbNo, name, &vg));
CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pDuplicate) {
*pDuplicate = false;
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&pStmt->targetTableName, tbFName);
STableMeta** pMeta = taosHashGet(pStmt->pSubTableHashObj, tbFName, strlen(tbFName));
if (NULL != pMeta) {
*pDuplicate = true;
int32_t code = ignoreUsingClause(pCxt, &pStmt->pSql);
if (TSDB_CODE_SUCCESS == code) {
return cloneTableMeta(*pMeta, &pStmt->pTableMeta);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) {
return getTableMetaImpl(pCxt, tbNo, name, false);
}
// pStmt->pSql -> field1_name, ...)
static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, SParsedDataColInfo* pColList,
SSchema* pSchema) {
col_id_t nCols = pColList->numOfCols;
static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) {
return getTableMetaImpl(pCxt, tbNo, name, true);
}
pColList->numOfBound = 0;
pColList->boundNullLen = 0;
memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols);
for (col_id_t i = 0; i < nCols; ++i) {
pColList->cols[i].valStat = VAL_STAT_NONE;
}
static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgInfo* pInfo) {
SParseContext* pBasicCtx = pCxt->pComCxt;
if (pBasicCtx->async) {
CHECK_CODE(getDbCfgFromCache(pCxt->pMetaCache, pDbFName, pInfo));
} else {
SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter,
.requestId = pBasicCtx->requestId,
.requestObjRefId = pBasicCtx->requestRid,
.mgmtEps = pBasicCtx->mgmtEpSet};
CHECK_CODE(catalogGetDBCfg(pBasicCtx->pCatalog, &conn, pDbFName, pInfo));
SToken token;
bool isOrdered = true;
col_id_t lastColIdx = -1; // last column found
while (1) {
NEXT_TOKEN(*pSql, token);
if (TK_NK_RP == token.type) {
break;
}
char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`)
strncpy(tmpTokenBuf, token.z, token.n);
token.z = tmpTokenBuf;
token.n = strdequote(token.z);
col_id_t t = lastColIdx + 1;
col_id_t index = insFindCol(&token, t, nCols, pSchema);
if (index < 0 && t > 0) {
index = insFindCol(&token, 0, t, pSchema);
isOrdered = false;
}
if (index < 0) {
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z);
}
if (pColList->cols[index].valStat == VAL_STAT_HAS) {
return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z);
}
lastColIdx = index;
pColList->cols[index].valStat = VAL_STAT_HAS;
pColList->boundColumns[pColList->numOfBound] = index;
++pColList->numOfBound;
switch (pSchema[t].type) {
case TSDB_DATA_TYPE_BINARY:
pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES);
break;
case TSDB_DATA_TYPE_NCHAR:
pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
break;
default:
pColList->boundNullLen += TYPE_BYTES[pSchema[t].type];
break;
}
}
pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED;
if (!isOrdered) {
pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo));
if (NULL == pColList->colIdxInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SBoundIdxInfo* pColIdx = pColList->colIdxInfo;
for (col_id_t i = 0; i < pColList->numOfBound; ++i) {
pColIdx[i].schemaColIdx = pColList->boundColumns[i];
pColIdx[i].boundIdx = i;
}
taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar);
for (col_id_t i = 0; i < pColList->numOfBound; ++i) {
pColIdx[i].finalIdx = i;
}
taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar);
}
if (pColList->numOfCols > pColList->numOfBound) {
memset(&pColList->boundColumns[pColList->numOfBound], 0,
sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
}
return TSDB_CODE_SUCCESS;
}
static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) {
int32_t index = 0;
SToken sToken;
int64_t interval;
int64_t ts = 0;
char* pTokenEnd = *end;
static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) {
int32_t index = 0;
int64_t interval;
int64_t ts = 0;
const char* pTokenEnd = *end;
if (pToken->type == TK_NOW) {
ts = taosGetTimestamp(timePrec);
......@@ -172,18 +295,17 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time
* time expression:
* e.g., now+12a, now-5h
*/
SToken valueToken;
index = 0;
sToken = tStrGetToken(pTokenEnd, &index, false);
SToken token = tStrGetToken(pTokenEnd, &index, false);
pTokenEnd += index;
if (sToken.type == TK_NK_MINUS || sToken.type == TK_NK_PLUS) {
if (token.type == TK_NK_MINUS || token.type == TK_NK_PLUS) {
index = 0;
valueToken = tStrGetToken(pTokenEnd, &index, false);
SToken valueToken = tStrGetToken(pTokenEnd, &index, false);
pTokenEnd += index;
if (valueToken.n < 2) {
return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", sToken.z);
return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", token.z);
}
char unit = 0;
......@@ -191,7 +313,7 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (sToken.type == TK_NK_PLUS) {
if (token.type == TK_NK_PLUS) {
ts += interval;
} else {
ts = ts - interval;
......@@ -204,92 +326,42 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) {
if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER &&
pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL &&
pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT &&
pToken->type != TK_NK_BIN) ||
(pToken->n == 0) || (pToken->type == TK_NK_RP)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
}
// Remove quotation marks
if (TK_NK_STRING == pToken->type) {
if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
}
int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
pToken->z = tmpTokenBuf;
pToken->n = len;
}
return TSDB_CODE_SUCCESS;
}
static bool isNullStr(SToken* pToken) {
return ((pToken->type == TK_NK_STRING) && (strlen(TSDB_DATA_NULL_STR_L) == pToken->n) &&
(strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
}
static bool isNullValue(int8_t dataType, SToken* pToken) {
return TK_NULL == pToken->type || (!IS_STR_DATA_TYPE(dataType) && isNullStr(pToken));
}
static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) {
errno = 0;
*value = taosStr2Double(pToken->z, endPtr);
// not a valid integer number, return error
if ((*endPtr - pToken->z) != pToken->n) {
return TK_NK_ILLEGAL;
}
return pToken->type;
}
static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf,
_row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) {
static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val,
SMsgBuf* pMsgBuf) {
int64_t iv;
uint64_t uv;
char* endptr = NULL;
int32_t code = checkAndTrimValue(pToken, tmpTokenBuf, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (isNullValue(pSchema->type, pToken)) {
if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z);
}
return func(pMsgBuf, NULL, 0, param);
return TSDB_CODE_SUCCESS;
}
if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) {
return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z);
}
// strcpy(val->colName, pSchema->name);
val->cid = pSchema->colId;
val->type = pSchema->type;
switch (pSchema->type) {
case TSDB_DATA_TYPE_BOOL: {
if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) {
if (strncmp(pToken->z, "true", pToken->n) == 0) {
return func(pMsgBuf, &TRUE_VALUE, pSchema->bytes, param);
*(int8_t*)(&val->i64) = TRUE_VALUE;
} else if (strncmp(pToken->z, "false", pToken->n) == 0) {
return func(pMsgBuf, &FALSE_VALUE, pSchema->bytes, param);
*(int8_t*)(&val->i64) = FALSE_VALUE;
} else {
return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
}
} else if (pToken->type == TK_NK_INTEGER) {
return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes,
param);
*(int8_t*)(&val->i64) = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE);
} else if (pToken->type == TK_NK_FLOAT) {
return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes,
param);
*(int8_t*)(&val->i64) = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE);
} else {
return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
}
break;
}
case TSDB_DATA_TYPE_TINYINT: {
......@@ -299,8 +371,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
return func(pMsgBuf, &tmpVal, pSchema->bytes, param);
*(int8_t*)(&val->i64) = iv;
break;
}
case TSDB_DATA_TYPE_UTINYINT: {
......@@ -309,8 +381,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
} else if (uv > UINT8_MAX) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)uv;
return func(pMsgBuf, &tmpVal, pSchema->bytes, param);
*(uint8_t*)(&val->i64) = uv;
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
......@@ -319,8 +391,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
} else if (!IS_VALID_SMALLINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z);
}
int16_t tmpVal = (int16_t)iv;
return func(pMsgBuf, &tmpVal, pSchema->bytes, param);
*(int16_t*)(&val->i64) = iv;
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
......@@ -329,8 +401,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
} else if (uv > UINT16_MAX) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z);
}
uint16_t tmpVal = (uint16_t)uv;
return func(pMsgBuf, &tmpVal, pSchema->bytes, param);
*(uint16_t*)(&val->i64) = uv;
break;
}
case TSDB_DATA_TYPE_INT: {
......@@ -339,8 +411,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
} else if (!IS_VALID_INT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z);
}
int32_t tmpVal = (int32_t)iv;
return func(pMsgBuf, &tmpVal, pSchema->bytes, param);
*(int32_t*)(&val->i64) = iv;
break;
}
case TSDB_DATA_TYPE_UINT: {
......@@ -349,22 +421,24 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
} else if (uv > UINT32_MAX) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z);
}
uint32_t tmpVal = (uint32_t)uv;
return func(pMsgBuf, &tmpVal, pSchema->bytes, param);
*(uint32_t*)(&val->i64) = uv;
break;
}
case TSDB_DATA_TYPE_BIGINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
}
return func(pMsgBuf, &iv, pSchema->bytes, param);
val->i64 = iv;
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z);
}
return func(pMsgBuf, &uv, pSchema->bytes, param);
*(uint64_t*)(&val->i64) = uv;
break;
}
case TSDB_DATA_TYPE_FLOAT: {
......@@ -376,8 +450,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
isnan(dv)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
}
float tmpVal = (float)dv;
return func(pMsgBuf, &tmpVal, pSchema->bytes, param);
*(float*)(&val->i64) = dv;
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
......@@ -388,7 +462,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
}
return func(pMsgBuf, &dv, pSchema->bytes, param);
*(double*)(&val->i64) = dv;
break;
}
case TSDB_DATA_TYPE_BINARY: {
......@@ -396,583 +472,750 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
return func(pMsgBuf, pToken->z, pToken->n, param);
val->pData = strdup(pToken->z);
val->nData = pToken->n;
break;
}
case TSDB_DATA_TYPE_NCHAR: {
return func(pMsgBuf, pToken->z, pToken->n, param);
}
case TSDB_DATA_TYPE_JSON: {
if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z);
int32_t output = 0;
void* p = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
if (errno == E2BIG) {
taosMemoryFree(p);
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
char buf[512] = {0};
snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
taosMemoryFree(p);
return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z);
}
return func(pMsgBuf, pToken->z, pToken->n, param);
val->pData = p;
val->nData = output;
break;
}
case TSDB_DATA_TYPE_TIMESTAMP: {
int64_t tmpVal;
if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z);
}
return func(pMsgBuf, &tmpVal, pSchema->bytes, param);
val->i64 = iv;
break;
}
}
return TSDB_CODE_FAILED;
return TSDB_CODE_SUCCESS;
}
// pSql -> tag1_name, ...)
static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) {
col_id_t nCols = pColList->numOfCols;
// input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ...
// output pStmt->pSql: TAGS (tag1_value, ...) ...
static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
SSchema* pTagsSchema = getTableTagSchema(pStmt->pTableMeta);
insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pStmt->pTableMeta));
pColList->numOfBound = 0;
pColList->boundNullLen = 0;
memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols);
for (col_id_t i = 0; i < nCols; ++i) {
pColList->cols[i].valStat = VAL_STAT_NONE;
SToken token;
int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index);
if (TK_NK_LP != token.type) {
return TSDB_CODE_SUCCESS;
}
SToken sToken;
bool isOrdered = true;
col_id_t lastColIdx = -1; // last column found
while (1) {
NEXT_TOKEN(pCxt->pSql, sToken);
pStmt->pSql += index;
return parseBoundColumns(pCxt, &pStmt->pSql, &pCxt->tags, pTagsSchema);
}
if (TK_NK_RP == sToken.type) {
break;
static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken,
SArray* pTagName, SArray* pTagVals, STag** pTag) {
if (!isNullValue(pTagSchema->type, pToken)) {
taosArrayPush(pTagName, pTagSchema->name);
}
if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z);
}
char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`)
strncpy(tmpTokenBuf, sToken.z, sToken.n);
sToken.z = tmpTokenBuf;
sToken.n = strdequote(sToken.z);
if (isNullValue(pTagSchema->type, pToken)) {
return tTagNew(pTagVals, 1, true, pTag);
} else {
return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg);
}
}
col_id_t t = lastColIdx + 1;
col_id_t index = insFindCol(&sToken, t, nCols, pSchema);
if (index < 0 && t > 0) {
index = insFindCol(&sToken, 0, t, pSchema);
isOrdered = false;
STagVal val = {0};
int32_t code =
parseTagToken(&pStmt->pSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code) {
taosArrayPush(pTagVals, &val);
}
return code;
}
static void buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) {
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&pStmt->targetTableName, tbFName);
char stbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&pStmt->usingTableName, stbFName);
insBuildCreateTbReq(&pStmt->createTblReq, tbFName, pTag, pStmt->pTableMeta->suid, stbFName, pTagName,
pStmt->pTableMeta->tableInfo.numOfTags);
}
static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) {
if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER &&
pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL &&
pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT &&
pToken->type != TK_NK_BIN) ||
(pToken->n == 0) || (pToken->type == TK_NK_RP)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
}
// Remove quotation marks
if (TK_NK_STRING == pToken->type) {
if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
}
if (index < 0) {
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, sToken.z);
int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
pToken->z = tmpTokenBuf;
pToken->n = len;
}
return TSDB_CODE_SUCCESS;
}
// pSql -> tag1_value, ...)
static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS;
SSchema* pSchema = getTableTagSchema(pStmt->pTableMeta);
SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal));
SArray* pTagName = taosArrayInit(8, TSDB_COL_NAME_LEN);
SToken token;
bool isParseBindParam = false;
bool isJson = false;
STag* pTag = NULL;
for (int i = 0; TSDB_CODE_SUCCESS == code && i < pCxt->tags.numOfBound; ++i) {
NEXT_TOKEN_WITH_PREV(pStmt->pSql, token);
if (token.type == TK_NK_QUESTION) {
isParseBindParam = true;
if (NULL == pCxt->pComCxt->pStmtCb) {
code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", token.z);
break;
}
continue;
}
if (pColList->cols[index].valStat == VAL_STAT_HAS) {
return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", sToken.z);
if (isParseBindParam) {
code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values");
break;
}
lastColIdx = index;
pColList->cols[index].valStat = VAL_STAT_HAS;
pColList->boundColumns[pColList->numOfBound] = index;
++pColList->numOfBound;
switch (pSchema[t].type) {
case TSDB_DATA_TYPE_BINARY:
pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES);
break;
case TSDB_DATA_TYPE_NCHAR:
pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
break;
default:
pColList->boundNullLen += TYPE_BYTES[pSchema[t].type];
break;
SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]];
isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON;
code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code) {
code = parseTagValue(pCxt, pStmt, pTagSchema, &token, pTagName, pTagVals, &pTag);
}
}
pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED;
if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) {
code = tTagNew(pTagVals, 1, false, &pTag);
}
if (!isOrdered) {
pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo));
if (NULL == pColList->colIdxInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SBoundIdxInfo* pColIdx = pColList->colIdxInfo;
for (col_id_t i = 0; i < pColList->numOfBound; ++i) {
pColIdx[i].schemaColIdx = pColList->boundColumns[i];
pColIdx[i].boundIdx = i;
if (TSDB_CODE_SUCCESS == code && !isParseBindParam) {
buildCreateTbReq(pStmt, pTag, pTagName);
pTag = NULL;
}
for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
STagVal* p = (STagVal*)taosArrayGet(pTagVals, i);
if (IS_VAR_DATA_TYPE(p->type)) {
taosMemoryFreeClear(p->pData);
}
taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar);
for (col_id_t i = 0; i < pColList->numOfBound; ++i) {
pColIdx[i].finalIdx = i;
}
taosArrayDestroy(pTagVals);
taosArrayDestroy(pTagName);
tTagFree(pTag);
return code;
}
// input pStmt->pSql: TAGS (tag1_value, ...) [table_options] ...
// output pStmt->pSql: [table_options] ...
static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
SToken token;
NEXT_TOKEN(pStmt->pSql, token);
if (TK_TAGS != token.type) {
return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", token.z);
}
NEXT_TOKEN(pStmt->pSql, token);
if (TK_NK_LP != token.type) {
return buildSyntaxErrMsg(&pCxt->msg, "( is expected", token.z);
}
int32_t code = parseTagsClauseImpl(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
NEXT_VALID_TOKEN(pStmt->pSql, token);
if (TK_NK_COMMA == token.type) {
code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_TAGS_NOT_MATCHED);
} else if (TK_NK_RP != token.type) {
code = buildSyntaxErrMsg(&pCxt->msg, ") is expected", token.z);
}
taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar);
}
return code;
}
if (pColList->numOfCols > pColList->numOfBound) {
memset(&pColList->boundColumns[pColList->numOfBound], 0,
sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
pStmt->pTableMeta->uid = pStmt->totalTbNum;
pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE;
STableMeta* pBackup = NULL;
if (TSDB_CODE_SUCCESS != cloneTableMeta(pStmt->pTableMeta, &pBackup)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&pStmt->targetTableName, tbFName);
return taosHashPut(pStmt->pSubTableHashObj, tbFName, strlen(tbFName), &pBackup, POINTER_BYTES);
}
static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
do {
int32_t index = 0;
SToken token;
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index);
if (TK_TTL == token.type) {
pStmt->pSql += index;
NEXT_TOKEN_WITH_PREV(pStmt->pSql, token);
if (TK_NK_INTEGER != token.type) {
return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z);
}
pStmt->createTblReq.ttl = taosStr2Int32(token.z, NULL, 10);
if (pStmt->createTblReq.ttl < 0) {
return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z);
}
} else if (TK_COMMENT == token.type) {
pStmt->pSql += index;
NEXT_TOKEN(pStmt->pSql, token);
if (TK_NK_STRING != token.type) {
return buildSyntaxErrMsg(&pCxt->msg, "Invalid option comment", token.z);
}
if (token.n >= TSDB_TB_COMMENT_LEN) {
return buildSyntaxErrMsg(&pCxt->msg, "comment too long", token.z);
}
int32_t len = trimString(token.z, token.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN);
pStmt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len);
if (NULL == pStmt->createTblReq.comment) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pStmt->createTblReq.commentLen = len;
} else {
break;
}
} while (1);
return TSDB_CODE_SUCCESS;
}
static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val,
SMsgBuf* pMsgBuf) {
int64_t iv;
uint64_t uv;
char* endptr = NULL;
// input pStmt->pSql:
// 1. [(tag1_name, ...)] ...
// 2. VALUES ... | FILE ...
// output pStmt->pSql:
// 1. [(field1_name, ...)]
// 2. VALUES ... | FILE ...
static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
if ('\0' == pStmt->usingTableName.tname[0]) {
return TSDB_CODE_SUCCESS;
}
if (isNullValue(pSchema->type, pToken)) {
if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z);
int32_t code = parseBoundTagsClause(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
code = parseTagsClause(pCxt, pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
code = parseTableOptions(pCxt, pStmt);
}
return code;
}
static int32_t checkAuth(SParseContext* pCxt, SName* pTbName) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTbName, dbFName);
SRequestConnInfo conn = {.pTrans = pCxt->pTransporter,
.requestId = pCxt->requestId,
.requestObjRefId = pCxt->requestRid,
.mgmtEps = pCxt->mgmtEpSet};
int32_t code = TSDB_CODE_SUCCESS;
bool pass = true;
if (pCxt->async) {
// todo replace with cached api
code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass);
} else {
code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass);
}
if (TSDB_CODE_SUCCESS == code && !pass) {
code = TSDB_CODE_PAR_PERMISSION_DENIED;
}
return code;
}
static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta,
bool* pMissCache) {
SParseContext* pComCxt = pCxt->pComCxt;
SRequestConnInfo conn = {.pTrans = pComCxt->pTransporter,
.requestId = pComCxt->requestId,
.requestObjRefId = pComCxt->requestRid,
.mgmtEps = pComCxt->mgmtEpSet};
int32_t code = TSDB_CODE_SUCCESS;
if (pComCxt->async) {
if (isStb) {
code = catalogGetCachedSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta);
} else {
code = catalogGetCachedTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta);
}
} else {
if (isStb) {
code = catalogGetSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta);
} else {
code = catalogGetTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta);
}
}
if (TSDB_CODE_SUCCESS == code) {
if (NULL == *pTableMeta) {
*pMissCache = true;
} else if (isStb && TSDB_SUPER_TABLE != (*pTableMeta)->tableType) {
code = buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed");
}
}
return code;
}
static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool isStb, bool* pMissCache) {
SRequestConnInfo conn = {.pTrans = pCxt->pTransporter,
.requestId = pCxt->requestId,
.requestObjRefId = pCxt->requestRid,
.mgmtEps = pCxt->mgmtEpSet};
int32_t code = TSDB_CODE_SUCCESS;
SVgroupInfo vg;
bool exists = true;
if (pCxt->async) {
code = catalogGetCachedTableHashVgroup(pCxt->pCatalog, &conn, &pStmt->targetTableName, &vg, &exists);
} else {
code = catalogGetTableHashVgroup(pCxt->pCatalog, &conn, &pStmt->targetTableName, &vg);
}
if (TSDB_CODE_SUCCESS == code) {
if (exists) {
if (isStb) {
pStmt->pTableMeta->vgId = vg.vgId;
}
code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg));
}
*pMissCache = !exists;
}
return code;
}
static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName);
if (TSDB_CODE_SUCCESS == code) {
code = getTableMeta(pCxt, &pStmt->targetTableName, false, &pStmt->pTableMeta, &pCxt->missCache);
}
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache);
}
return code;
}
static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) {
return insCreateSName(&pStmt->usingTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
}
static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName);
if (TSDB_CODE_SUCCESS == code) {
code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache);
}
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = getTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache);
}
return code;
}
static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
SToken token;
NEXT_TOKEN(pStmt->pSql, token);
int32_t code = preParseUsingTableName(pCxt, pStmt, &token);
if (TSDB_CODE_SUCCESS == code) {
code = getUsingTableSchema(pCxt, pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
code = storeTableMeta(pCxt, pStmt);
}
return code;
}
// input pStmt->pSql:
// 1(care). [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ...
// 2. VALUES ... | FILE ...
// output pStmt->pSql:
// 1. [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ...
// 2. VALUES ... | FILE ...
static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
SToken token;
int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index);
if (TK_USING != token.type) {
return getTargetTableSchema(pCxt, pStmt);
}
// pStmt->pSql -> stb_name [(tag1_name, ...)
pStmt->pSql += index;
bool duplicate = false;
int32_t code = parseDuplicateUsingClause(pCxt, pStmt, &duplicate);
if (TSDB_CODE_SUCCESS == code && !duplicate) {
return parseUsingTableNameImpl(pCxt, pStmt);
}
return code;
}
static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) {
return insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
}
// input pStmt->pSql:
// 1(care). [(field1_name, ...)] ...
// 2. [ USING ... ] ...
// 3. VALUES ... | FILE ...
// output pStmt->pSql:
// 1. [ USING ... ] ...
// 2. VALUES ... | FILE ...
static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
SToken token;
int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index);
if (TK_NK_LP != token.type) {
return TSDB_CODE_SUCCESS;
}
// strcpy(val->colName, pSchema->name);
val->cid = pSchema->colId;
val->type = pSchema->type;
// pStmt->pSql -> field1_name, ...)
pStmt->pSql += index;
pStmt->pBoundCols = pStmt->pSql;
return skipParentheses(pCxt, &pStmt->pSql);
}
static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks** pDataBuf) {
if (pCxt->pComCxt->async) {
return insGetDataBlockFromList(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid),
TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk),
getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL,
&pStmt->createTblReq);
}
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&pStmt->targetTableName, tbFName);
return insGetDataBlockFromList(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE,
sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta,
pDataBuf, NULL, &pStmt->createTblReq);
}
static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt,
STableDataBlocks* pDataBuf) {
SToken token;
int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index);
if (TK_NK_LP == token.type) {
pStmt->pSql += index;
if (NULL != pStmt->pBoundCols) {
return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z);
}
// pStmt->pSql -> field1_name, ...)
return parseBoundColumns(pCxt, &pStmt->pSql, &pDataBuf->boundColumnInfo, getTableColumnSchema(pStmt->pTableMeta));
}
if (NULL != pStmt->pBoundCols) {
return parseBoundColumns(pCxt, &pStmt->pBoundCols, &pDataBuf->boundColumnInfo,
getTableColumnSchema(pStmt->pTableMeta));
}
return TSDB_CODE_SUCCESS;
}
// input pStmt->pSql:
// 1. [(tag1_name, ...)] ...
// 2. VALUES ... | FILE ...
// output pStmt->pSql: VALUES ... | FILE ...
static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt,
STableDataBlocks** pDataBuf) {
int32_t code = parseUsingClauseBottom(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
code = getTableDataBlocks(pCxt, pStmt, pDataBuf);
}
if (TSDB_CODE_SUCCESS == code) {
code = parseBoundColumnsClause(pCxt, pStmt, *pDataBuf);
}
return code;
}
// input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ...
// output pStmt->pSql:
// 1. [(tag1_name, ...)] ...
// 2. VALUES ... | FILE ...
static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) {
int32_t code = preParseTargetTableName(pCxt, pStmt, pTbName);
if (TSDB_CODE_SUCCESS == code) {
// option: [(field1_name, ...)]
code = preParseBoundColumnsClause(pCxt, pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
// option: [USING stb_name]
code = parseUsingTableName(pCxt, pStmt);
}
return code;
}
static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema,
int16_t timePrec, _row_append_fn_t func, void* param) {
int64_t iv;
uint64_t uv;
char* endptr = NULL;
switch (pSchema->type) {
case TSDB_DATA_TYPE_BOOL: {
if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) {
if (strncmp(pToken->z, "true", pToken->n) == 0) {
*(int8_t*)(&val->i64) = TRUE_VALUE;
return func(&pCxt->msg, &TRUE_VALUE, pSchema->bytes, param);
} else if (strncmp(pToken->z, "false", pToken->n) == 0) {
*(int8_t*)(&val->i64) = FALSE_VALUE;
return func(&pCxt->msg, &FALSE_VALUE, pSchema->bytes, param);
} else {
return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z);
}
} else if (pToken->type == TK_NK_INTEGER) {
*(int8_t*)(&val->i64) = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE);
return func(&pCxt->msg, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE),
pSchema->bytes, param);
} else if (pToken->type == TK_NK_FLOAT) {
*(int8_t*)(&val->i64) = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE);
return func(&pCxt->msg, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes,
param);
} else {
return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z);
}
break;
}
case TSDB_DATA_TYPE_TINYINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z);
} else if (!IS_VALID_TINYINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z);
}
*(int8_t*)(&val->i64) = iv;
break;
uint8_t tmpVal = (uint8_t)iv;
return func(&pCxt->msg, &tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_UTINYINT: {
if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z);
} else if (uv > UINT8_MAX) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z);
}
*(uint8_t*)(&val->i64) = uv;
break;
uint8_t tmpVal = (uint8_t)uv;
return func(&pCxt->msg, &tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_SMALLINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z);
} else if (!IS_VALID_SMALLINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z);
}
*(int16_t*)(&val->i64) = iv;
break;
int16_t tmpVal = (int16_t)iv;
return func(&pCxt->msg, &tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_USMALLINT: {
if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z);
} else if (uv > UINT16_MAX) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z);
}
*(uint16_t*)(&val->i64) = uv;
break;
uint16_t tmpVal = (uint16_t)uv;
return func(&pCxt->msg, &tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_INT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z);
} else if (!IS_VALID_INT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z);
}
*(int32_t*)(&val->i64) = iv;
break;
int32_t tmpVal = (int32_t)iv;
return func(&pCxt->msg, &tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_UINT: {
if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z);
} else if (uv > UINT32_MAX) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z);
}
*(uint32_t*)(&val->i64) = uv;
break;
uint32_t tmpVal = (uint32_t)uv;
return func(&pCxt->msg, &tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_BIGINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z);
}
val->i64 = iv;
break;
return func(&pCxt->msg, &iv, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_UBIGINT: {
if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z);
}
*(uint64_t*)(&val->i64) = uv;
break;
return func(&pCxt->msg, &uv, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_FLOAT: {
double dv;
if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
isnan(dv)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
}
*(float*)(&val->i64) = dv;
break;
float tmpVal = (float)dv;
return func(&pCxt->msg, &tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_DOUBLE: {
double dv;
if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z);
}
*(double*)(&val->i64) = dv;
break;
return func(&pCxt->msg, &dv, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_BINARY: {
// Too long values will raise the invalid sql error message
if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
val->pData = strdup(pToken->z);
val->nData = pToken->n;
break;
}
case TSDB_DATA_TYPE_NCHAR: {
int32_t output = 0;
void* p = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
if (errno == E2BIG) {
taosMemoryFree(p);
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
char buf[512] = {0};
snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
taosMemoryFree(p);
return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z);
}
val->pData = p;
val->nData = output;
break;
}
case TSDB_DATA_TYPE_TIMESTAMP: {
if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z);
}
val->i64 = iv;
break;
}
}
return TSDB_CODE_SUCCESS;
}
// pSql -> tag1_value, ...)
static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal));
SArray* tagName = taosArrayInit(8, TSDB_COL_NAME_LEN);
SToken sToken;
bool isParseBindParam = false;
bool isJson = false;
STag* pTag = NULL;
for (int i = 0; i < pCxt->tags.numOfBound; ++i) {
NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
if (sToken.type == TK_NK_QUESTION) {
isParseBindParam = true;
if (NULL == pCxt->pStmtCb) {
code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z);
break;
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
continue;
}
if (isParseBindParam) {
code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values");
break;
}
SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]];
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // todo this can be optimize with parse column
code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code) {
if (!isNullValue(pTagSchema->type, &sToken)) {
taosArrayPush(tagName, pTagSchema->name);
}
if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
isJson = true;
if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z);
break;
}
if (isNullValue(pTagSchema->type, &sToken)) {
code = tTagNew(pTagVals, 1, true, &pTag);
} else {
code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg);
}
} else {
STagVal val = {0};
code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code) {
taosArrayPush(pTagVals, &val);
}
}
}
if (TSDB_CODE_SUCCESS != code) {
break;
return func(&pCxt->msg, pToken->z, pToken->n, param);
}
}
if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) {
code = tTagNew(pTagVals, 1, false, &pTag);
}
if (TSDB_CODE_SUCCESS == code && !isParseBindParam) {
insBuildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName,
pCxt->pTableMeta->tableInfo.numOfTags);
pTag = NULL;
}
for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
STagVal* p = (STagVal*)taosArrayGet(pTagVals, i);
if (IS_VAR_DATA_TYPE(p->type)) {
taosMemoryFreeClear(p->pData);
}
}
taosArrayDestroy(pTagVals);
taosArrayDestroy(tagName);
tTagFree(pTag);
return code;
}
static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, int32_t tbNo, SName* pTableName,
const char* pName, int32_t len, STableMeta* pMeta) {
SVgroupInfo vg;
CHECK_CODE(getTableVgroup(pCxt, tbNo, pTableName, &vg));
CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
pMeta->uid = tbNo;
pMeta->vgId = vg.vgId;
pMeta->tableType = TSDB_CHILD_TABLE;
STableMeta* pBackup = NULL;
if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES);
}
static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) {
SToken sToken;
int32_t expectRightParenthesis = 1;
while (1) {
NEXT_TOKEN(pCxt->pSql, sToken);
if (TK_NK_LP == sToken.type) {
++expectRightParenthesis;
} else if (TK_NK_RP == sToken.type && 0 == --expectRightParenthesis) {
break;
}
if (0 == sToken.n) {
return buildSyntaxErrMsg(&pCxt->msg, ") expected", NULL);
case TSDB_DATA_TYPE_NCHAR: {
return func(&pCxt->msg, pToken->z, pToken->n, param);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t skipBoundColumns(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); }
static int32_t ignoreBoundColumns(SInsertParseContext* pCxt) {
SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL};
int32_t code = skipBoundColumns(&cxt);
pCxt->pSql = cxt.pSql;
return code;
}
static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt);
// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
static int32_t ignoreAutoCreateTableClause(SInsertParseContext* pCxt) {
SToken sToken;
NEXT_TOKEN(pCxt->pSql, sToken);
SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL};
int32_t code = skipUsingClause(&cxt);
pCxt->pSql = cxt.pSql;
return code;
}
static int32_t parseTableOptions(SInsertParseContext* pCxt) {
do {
int32_t index = 0;
SToken sToken;
NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index);
if (TK_TTL == sToken.type) {
pCxt->pSql += index;
NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
if (TK_NK_INTEGER != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z);
}
pCxt->createTblReq.ttl = taosStr2Int32(sToken.z, NULL, 10);
if (pCxt->createTblReq.ttl < 0) {
return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z);
}
} else if (TK_COMMENT == sToken.type) {
pCxt->pSql += index;
NEXT_TOKEN(pCxt->pSql, sToken);
if (TK_NK_STRING != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, "Invalid option comment", sToken.z);
}
if (sToken.n >= TSDB_TB_COMMENT_LEN) {
return buildSyntaxErrMsg(&pCxt->msg, "comment too long", sToken.z);
}
int32_t len = trimString(sToken.z, sToken.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN);
pCxt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len);
if (NULL == pCxt->createTblReq.comment) {
return TSDB_CODE_OUT_OF_MEMORY;
case TSDB_DATA_TYPE_JSON: {
if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z);
}
pCxt->createTblReq.commentLen = len;
} else {
break;
return func(&pCxt->msg, pToken->z, pToken->n, param);
}
} while (1);
return TSDB_CODE_SUCCESS;
}
case TSDB_DATA_TYPE_TIMESTAMP: {
int64_t tmpVal;
if (parseTime(pSql, pToken, timePrec, &tmpVal, &pCxt->msg) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z);
}
// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* tbFName) {
int32_t len = strlen(tbFName);
STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len);
if (NULL != pMeta) {
CHECK_CODE(ignoreAutoCreateTableClause(pCxt));
return cloneTableMeta(*pMeta, &pCxt->pTableMeta);
return func(&pCxt->msg, &tmpVal, pSchema->bytes, param);
}
}
SToken sToken;
// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
NEXT_TOKEN(pCxt->pSql, sToken);
return TSDB_CODE_FAILED;
}
SName sname;
CHECK_CODE(insCreateSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg));
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&sname, dbFName);
strcpy(pCxt->sTableName, sname.tname);
static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema,
int16_t timePrec, _row_append_fn_t func, void* param) {
int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) {
if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z);
}
CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname));
if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) {
return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed");
return func(&pCxt->msg, NULL, 0, param);
}
CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, tbNo, name, tbFName, len, pCxt->pTableMeta));
SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta);
insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta));
// pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...)
NEXT_TOKEN(pCxt->pSql, sToken);
if (TK_NK_LP == sToken.type) {
CHECK_CODE(parseBoundColumns(pCxt, &pCxt->tags, pTagsSchema));
NEXT_TOKEN(pCxt->pSql, sToken);
if (TSDB_CODE_SUCCESS == code && IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z);
}
if (TK_TAGS != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z);
}
// pSql -> (tag1_value, ...)
NEXT_TOKEN(pCxt->pSql, sToken);
if (TK_NK_LP != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z);
}
CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision, name->tname));
NEXT_VALID_TOKEN(pCxt->pSql, sToken);
if (TK_NK_COMMA == sToken.type) {
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_TAGS_NOT_MATCHED);
} else if (TK_NK_RP != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z);
if (TSDB_CODE_SUCCESS == code) {
code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, func, param);
}
return parseTableOptions(pCxt);
return code;
}
static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, bool* gotRow,
char* tmpTokenBuf) {
SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo;
SRowBuilder* pBuilder = &pDataBlocks->rowBuilder;
STSRow* row = (STSRow*)(pDataBlocks->pData + pDataBlocks->size); // skip the SSubmitBlk header
static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataBlocks* pDataBuf, bool* pGotRow,
SToken* pToken) {
SRowBuilder* pBuilder = &pDataBuf->rowBuilder;
STSRow* row = (STSRow*)(pDataBuf->pData + pDataBuf->size); // skip the SSubmitBlk header
SParsedDataColInfo* pCols = &pDataBuf->boundColumnInfo;
bool isParseBindParam = false;
SSchema* pSchemas = getTableColumnSchema(pDataBuf->pTableMeta);
SMemParam param = {.rb = pBuilder};
tdSRowResetBuf(pBuilder, row);
bool isParseBindParam = false;
SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta);
SMemParam param = {.rb = pBuilder};
SToken sToken = {0};
int32_t code = tdSRowResetBuf(pBuilder, row);
// 1. set the parsed value from sql string
for (int i = 0; i < spd->numOfBound; ++i) {
NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
SSchema* pSchema = &schema[spd->boundColumns[i]];
for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) {
NEXT_TOKEN_WITH_PREV(*pSql, *pToken);
SSchema* pSchema = &pSchemas[pCols->boundColumns[i]];
if (sToken.type == TK_NK_QUESTION) {
if (pToken->type == TK_NK_QUESTION) {
isParseBindParam = true;
if (NULL == pCxt->pStmtCb) {
return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z);
if (NULL == pCxt->pComCxt->pStmtCb) {
code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pToken->z);
}
continue;
}
if (TK_NK_RP == sToken.type) {
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
if (TSDB_CODE_SUCCESS == code && TK_NK_RP == pToken->type) {
code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
}
if (isParseBindParam) {
return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values");
if (TSDB_CODE_SUCCESS == code && isParseBindParam) {
code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values");
}
param.schema = pSchema;
insGetSTSRowAppendInfo(pBuilder->rowType, spd, i, &param.toffset, &param.colIdx);
CHECK_CODE(
parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, insMemRowAppend, &param, &pCxt->msg));
if (TSDB_CODE_SUCCESS == code) {
param.schema = pSchema;
insGetSTSRowAppendInfo(pBuilder->rowType, pCols, i, &param.toffset, &param.colIdx);
code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pDataBuf->pTableMeta).precision, insMemRowAppend,
&param);
}
if (i < spd->numOfBound - 1) {
NEXT_VALID_TOKEN(pCxt->pSql, sToken);
if (TK_NK_COMMA != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, ", expected", sToken.z);
if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) {
NEXT_VALID_TOKEN(*pSql, *pToken);
if (TK_NK_COMMA != pToken->type) {
code = buildSyntaxErrMsg(&pCxt->msg, ", expected", pToken->z);
}
}
}
TSKEY tsKey = TD_ROW_KEY(row);
insCheckTimestamp(pDataBlocks, (const char*)&tsKey);
if (TSDB_CODE_SUCCESS == code) {
TSKEY tsKey = TD_ROW_KEY(row);
code = insCheckTimestamp(pDataBuf, (const char*)&tsKey);
}
if (!isParseBindParam) {
if (TSDB_CODE_SUCCESS == code && !isParseBindParam) {
// set the null value for the columns that do not assign values
if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
if ((pCols->numOfBound < pCols->numOfCols) && TD_IS_TP_ROW(row)) {
pBuilder->hasNone = true;
}
tdSRowEnd(pBuilder);
*gotRow = true;
*pGotRow = true;
#ifdef TD_DEBUG_PRINT_ROW
STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1);
......@@ -981,8 +1224,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
#endif
}
// *len = pBuilder->extendedRowSize;
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, int32_t* numOfRows) {
......@@ -1014,84 +1256,81 @@ static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize,
}
// pSql -> (field1_value, ...) [(field1_value2, ...) ...]
static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows) {
STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta);
int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock);
CHECK_CODE(
insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo));
(*numOfRows) = 0;
// char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \"
SToken sToken;
while (1) {
static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf,
int32_t maxRows, int32_t* pNumOfRows, SToken* pToken) {
int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo);
int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf);
(*pNumOfRows) = 0;
while (TSDB_CODE_SUCCESS == code) {
int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index);
if (TK_NK_LP != sToken.type) {
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, *pToken, index);
if (TK_NK_LP != pToken->type) {
break;
}
pCxt->pSql += index;
pStmt->pSql += index;
if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) {
int32_t tSize;
CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize));
ASSERT(tSize >= maxRows);
maxRows = tSize;
if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) {
code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows);
}
bool gotRow = false;
CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, pCxt->tmpTokenBuf));
if (gotRow) {
pDataBlock->size += extendedRowSize; // len;
if (TSDB_CODE_SUCCESS == code) {
code = parseOneRow(pCxt, &pStmt->pSql, pDataBuf, &gotRow, pToken);
}
NEXT_VALID_TOKEN(pCxt->pSql, sToken);
if (TK_NK_COMMA == sToken.type) {
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
} else if (TK_NK_RP != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z);
if (TSDB_CODE_SUCCESS == code) {
NEXT_VALID_TOKEN(pStmt->pSql, *pToken);
if (TK_NK_COMMA == pToken->type) {
code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
} else if (TK_NK_RP != pToken->type) {
code = buildSyntaxErrMsg(&pCxt->msg, ") expected", pToken->z);
}
}
if (gotRow) {
(*numOfRows)++;
if (TSDB_CODE_SUCCESS == code && gotRow) {
pDataBuf->size += extendedRowSize;
(*pNumOfRows)++;
}
}
if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) &&
(!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
}
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* dataBuf) {
int32_t maxNumOfRows;
CHECK_CODE(allocateMemIfNeed(dataBuf, insGetExtendedRowSize(dataBuf), &maxNumOfRows));
// VALUES (field1_value, ...) [(field1_value2, ...) ...]
static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf,
SToken* pToken) {
int32_t maxNumOfRows = 0;
int32_t numOfRows = 0;
CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows));
SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData);
if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, dataBuf, numOfRows)) {
return buildInvalidOperationMsg(&pCxt->msg,
"too many rows in sql, total number of rows should be less than INT32_MAX");
int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows);
if (TSDB_CODE_SUCCESS == code) {
code = parseValues(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows, pToken);
}
dataBuf->numOfTables = 1;
pCxt->totalNum += numOfRows;
return TSDB_CODE_SUCCESS;
if (TSDB_CODE_SUCCESS == code) {
code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg);
}
if (TSDB_CODE_SUCCESS == code) {
pDataBuf->numOfTables = 1;
pStmt->totalRowsNum += numOfRows;
pStmt->totalTbNum += 1;
TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_INSERT);
}
return code;
}
static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataBlocks* pDataBlock, int maxRows,
int32_t* numOfRows) {
STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta);
int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock);
CHECK_CODE(
insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo));
static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf,
int maxRows, int32_t* pNumOfRows) {
int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo);
(*numOfRows) = 0;
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \"
int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf);
(*pNumOfRows) = 0;
char* pLine = NULL;
int64_t readLen = 0;
while ((readLen = taosGetLineFile(fp, &pLine)) != -1) {
while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pStmt->fp, &pLine)) != -1) {
if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) {
pLine[--readLen] = '\0';
}
......@@ -1100,588 +1339,516 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB
continue;
}
if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) {
int32_t tSize;
CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize));
ASSERT(tSize >= maxRows);
maxRows = tSize;
if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) {
code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows);
}
strtolower(pLine, pLine);
char* pRawSql = pCxt->pSql;
pCxt->pSql = pLine;
bool gotRow = false;
int32_t code = parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, tmpTokenBuf);
if (TSDB_CODE_SUCCESS != code) {
pCxt->pSql = pRawSql;
return code;
bool gotRow = false;
if (TSDB_CODE_SUCCESS == code) {
SToken token;
strtolower(pLine, pLine);
code = parseOneRow(pCxt, (const char**)&pLine, pDataBuf, &gotRow, &token);
}
if (gotRow) {
pDataBlock->size += extendedRowSize; // len;
(*numOfRows)++;
if (TSDB_CODE_SUCCESS == code && gotRow) {
pDataBuf->size += extendedRowSize;
(*pNumOfRows)++;
}
pCxt->pSql = pRawSql;
if (pDataBlock->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) {
if (TSDB_CODE_SUCCESS == code && pDataBuf->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) {
break;
}
}
if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) &&
(!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
}
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t parseDataFromFileAgain(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName,
STableDataBlocks* dataBuf) {
int32_t maxNumOfRows;
CHECK_CODE(allocateMemIfNeed(dataBuf, insGetExtendedRowSize(dataBuf), &maxNumOfRows));
static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) {
int32_t maxNumOfRows = 0;
int32_t numOfRows = 0;
CHECK_CODE(parseCsvFile(pCxt, pCxt->pComCxt->csvCxt.fp, dataBuf, maxNumOfRows, &numOfRows));
SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData);
if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, dataBuf, numOfRows)) {
return buildInvalidOperationMsg(&pCxt->msg,
"too many rows in sql, total number of rows should be less than INT32_MAX");
int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows);
if (TSDB_CODE_SUCCESS == code) {
code = parseCsvFile(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows);
}
if (!taosEOFFile(pCxt->pComCxt->csvCxt.fp)) {
pCxt->pComCxt->needMultiParse = true;
pCxt->pComCxt->csvCxt.tableNo = tableNo;
memcpy(&pCxt->pComCxt->csvCxt.tableName, pTableName, sizeof(SName));
pCxt->pComCxt->csvCxt.pLastSqlPos = pCxt->pSql;
if (TSDB_CODE_SUCCESS == code) {
code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg);
}
if (TSDB_CODE_SUCCESS == code) {
pDataBuf->numOfTables = 1;
pStmt->totalRowsNum += numOfRows;
pStmt->totalTbNum += 1;
TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT);
if (taosEOFFile(pStmt->fp)) {
taosCloseFile(&pStmt->fp);
} else {
parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId);
}
}
dataBuf->numOfTables = 1;
pCxt->totalNum += numOfRows;
return TSDB_CODE_SUCCESS;
}
static int32_t parseDataFromFile(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, SToken filePath,
STableDataBlocks* dataBuf) {
static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pFilePath,
STableDataBlocks* pDataBuf) {
char filePathStr[TSDB_FILENAME_LEN] = {0};
if (TK_NK_STRING == filePath.type) {
trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr));
if (TK_NK_STRING == pFilePath->type) {
trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr));
} else {
strncpy(filePathStr, filePath.z, filePath.n);
strncpy(filePathStr, pFilePath->z, pFilePath->n);
}
pCxt->pComCxt->csvCxt.fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == pCxt->pComCxt->csvCxt.fp) {
pStmt->fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == pStmt->fp) {
return TAOS_SYSTEM_ERROR(errno);
}
return parseDataFromFileAgain(pCxt, tableNo, pTableName, dataBuf);
return parseDataFromFileImpl(pCxt, pStmt, pDataBuf);
}
static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
if (!pCxt->pComCxt->needMultiParse) {
taosCloseFile(&pCxt->pComCxt->csvCxt.fp);
static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf,
SToken* pToken) {
NEXT_TOKEN(pStmt->pSql, *pToken);
if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) {
return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z);
}
taosMemoryFreeClear(pCxt->pTableMeta);
destroyBoundColumnInfo(&pCxt->tags);
tdDestroySVCreateTbReq(&pCxt->createTblReq);
return parseDataFromFile(pCxt, pStmt, pToken, pDataBuf);
}
static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); }
static void destroyInsertParseContext(SInsertParseContext* pCxt) {
destroyInsertParseContextForTable(pCxt);
taosHashCleanup(pCxt->pVgroupsHashObj);
taosHashCleanup(pCxt->pSubTableHashObj);
taosHashCleanup(pCxt->pTableNameHashObj);
taosHashCleanup(pCxt->pDbFNameHashObj);
insDestroyBlockHashmap(pCxt->pTableBlockHashObj);
insDestroyBlockArrayList(pCxt->pVgDataBlocks);
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) {
SToken token;
NEXT_TOKEN(pStmt->pSql, token);
switch (token.type) {
case TK_VALUES:
return parseValuesClause(pCxt, pStmt, pDataBuf, &token);
case TK_FILE:
return parseFileClause(pCxt, pStmt, pDataBuf, &token);
default:
break;
}
return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z);
}
static int32_t parseTableName(SInsertParseContext* pCxt, SToken* pTbnameToken, SName* pName, char* pDbFName,
char* pTbFName) {
int32_t code = insCreateSName(pName, pTbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code) {
tNameExtractFullName(pName, pTbFName);
code = taosHashPut(pCxt->pTableNameHashObj, pTbFName, strlen(pTbFName), pName, sizeof(SName));
}
// input pStmt->pSql:
// 1. [(tag1_name, ...)] ...
// 2. VALUES ... | FILE ...
static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
STableDataBlocks* pDataBuf = NULL;
int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pDataBuf);
if (TSDB_CODE_SUCCESS == code) {
tNameGetFullDbName(pName, pDbFName);
code = taosHashPut(pCxt->pDbFNameHashObj, pDbFName, strlen(pDbFName), pDbFName, TSDB_DB_FNAME_LEN);
code = parseDataClause(pCxt, pStmt, pDataBuf);
}
return code;
}
// tb_name
// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
// [(field1_name, ...)]
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
// [...];
static int32_t parseInsertBody(SInsertParseContext* pCxt) {
int32_t tbNum = 0;
SName name;
char tbFName[TSDB_TABLE_FNAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
bool autoCreateTbl = false;
// for each table
while (1) {
SToken sToken;
char* tbName = NULL;
// pSql -> tb_name ...
NEXT_TOKEN(pCxt->pSql, sToken);
// no data in the sql string anymore.
if (sToken.n == 0) {
if (sToken.type && pCxt->pSql[0]) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z);
}
// input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ...
static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) {
int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName);
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = parseInsertTableClauseBottom(pCxt, pStmt);
}
return code;
}
if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) &&
!pCxt->pComCxt->needMultiParse) {
return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");
}
break;
static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName,
bool* pHasData) {
// no data in the sql string anymore.
if (0 == pTbName->n) {
if (0 != pTbName->type && '\0' != pStmt->pSql[0]) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", pTbName->z);
}
if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && tbNum > 0) {
return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt");
if (0 == pStmt->totalRowsNum && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");
}
destroyInsertParseContextForTable(pCxt);
if (TK_NK_QUESTION == sToken.type) {
if (pCxt->pStmtCb) {
CHECK_CODE((*pCxt->pStmtCb->getTbNameFn)(pCxt->pStmtCb->pStmt, &tbName));
sToken.z = tbName;
sToken.n = strlen(tbName);
} else {
return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z);
}
}
*pHasData = false;
return TSDB_CODE_SUCCESS;
}
SToken tbnameToken = sToken;
NEXT_TOKEN(pCxt->pSql, sToken);
if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && pStmt->totalTbNum > 0) {
return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt");
}
if (!pCxt->pComCxt->async || TK_USING == sToken.type) {
CHECK_CODE(parseTableName(pCxt, &tbnameToken, &name, dbFName, tbFName));
if (TK_NK_QUESTION == pTbName->type) {
if (NULL == pCxt->pComCxt->pStmtCb) {
return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pTbName->z);
}
bool existedUsing = false;
// USING clause
if (TK_USING == sToken.type) {
existedUsing = true;
CHECK_CODE(parseUsingClause(pCxt, tbNum, &name, tbFName));
NEXT_TOKEN(pCxt->pSql, sToken);
autoCreateTbl = true;
char* tbName = NULL;
int32_t code = (*pCxt->pComCxt->pStmtCb->getTbNameFn)(pCxt->pComCxt->pStmtCb->pStmt, &tbName);
if (TSDB_CODE_SUCCESS == code) {
pTbName->z = tbName;
pTbName->n = strlen(tbName);
} else {
return code;
}
}
char* pBoundColsStart = NULL;
if (TK_NK_LP == sToken.type) {
// pSql -> field1_name, ...)
pBoundColsStart = pCxt->pSql;
CHECK_CODE(ignoreBoundColumns(pCxt));
NEXT_TOKEN(pCxt->pSql, sToken);
}
*pHasData = true;
return TSDB_CODE_SUCCESS;
}
if (TK_USING == sToken.type) {
if (pCxt->pComCxt->async) {
CHECK_CODE(parseTableName(pCxt, &tbnameToken, &name, dbFName, tbFName));
}
CHECK_CODE(parseUsingClause(pCxt, tbNum, &name, tbFName));
NEXT_TOKEN(pCxt->pSql, sToken);
autoCreateTbl = true;
} else if (!existedUsing) {
CHECK_CODE(getTableMeta(pCxt, tbNum, &name));
if (TSDB_SUPER_TABLE == pCxt->pTableMeta->tableType) {
return buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported");
}
}
static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags));
if (NULL == tags) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memcpy(tags, &pCxt->tags, sizeof(pCxt->tags));
STableDataBlocks* dataBuf = NULL;
if (pCxt->pComCxt->async) {
CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid,
sizeof(pCxt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk),
getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL,
&pCxt->createTblReq));
} else {
CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE,
sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta,
&dataBuf, NULL, &pCxt->createTblReq));
}
SStmtCallback* pStmtCb = pCxt->pComCxt->pStmtCb;
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&pStmt->targetTableName, tbFName);
char stbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&pStmt->usingTableName, stbFName);
int32_t code =
(*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, tbFName, '\0' != pStmt->usingTableName.tname[0],
pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj, stbFName);
if (NULL != pBoundColsStart) {
char* pCurrPos = pCxt->pSql;
pCxt->pSql = pBoundColsStart;
CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)));
pCxt->pSql = pCurrPos;
}
memset(&pCxt->tags, 0, sizeof(pCxt->tags));
pStmt->pVgroupsHashObj = NULL;
pStmt->pTableBlockHashObj = NULL;
return code;
}
if (TK_VALUES == sToken.type) {
// pSql -> (field1_value, ...) [(field1_value2, ...) ...]
CHECK_CODE(parseValuesClause(pCxt, dataBuf));
TSDB_QUERY_SET_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_INSERT);
static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) {
return setStmtInfo(pCxt, pStmt);
}
tbNum++;
continue;
}
// merge according to vgId
int32_t code = TSDB_CODE_SUCCESS;
if (taosHashGetSize(pStmt->pTableBlockHashObj) > 0) {
code = insMergeTableDataBlocks(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks);
}
if (TSDB_CODE_SUCCESS == code) {
code = insBuildOutput(pStmt);
}
return code;
}
// FILE csv_file_path
if (TK_FILE == sToken.type) {
// pSql -> csv_file_path
NEXT_TOKEN(pCxt->pSql, sToken);
if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) {
return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z);
}
CHECK_CODE(parseDataFromFile(pCxt, tbNum, &name, sToken, dataBuf));
pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT;
static void destroyEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
destroyBoundColumnInfo(&pCxt->tags);
taosMemoryFreeClear(pStmt->pTableMeta);
tdDestroySVCreateTbReq(&pStmt->createTblReq);
}
tbNum++;
if (!pCxt->pComCxt->needMultiParse) {
continue;
} else {
parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId);
break;
}
// tb_name
// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
// [(field1_name, ...)]
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
// [...];
static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
SToken token;
int32_t code = TSDB_CODE_SUCCESS;
bool hasData = true;
// for each table
while (TSDB_CODE_SUCCESS == code && hasData && !pCxt->missCache) {
destroyEnvPreTable(pCxt, pStmt);
// pStmt->pSql -> tb_name ...
NEXT_TOKEN(pStmt->pSql, token);
code = checkTableClauseFirstToken(pCxt, pStmt, &token, &hasData);
if (TSDB_CODE_SUCCESS == code && hasData) {
code = parseInsertTableClause(pCxt, pStmt, &token);
}
return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z);
}
parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pStmt->totalRowsNum);
if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) {
SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags));
if (NULL == tags) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memcpy(tags, &pCxt->tags, sizeof(pCxt->tags));
(*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl,
pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj, pCxt->sTableName);
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = parseInsertBodyBottom(pCxt, pStmt);
}
return code;
}
memset(&pCxt->tags, 0, sizeof(pCxt->tags));
pCxt->pVgroupsHashObj = NULL;
pCxt->pTableBlockHashObj = NULL;
static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); }
return TSDB_CODE_SUCCESS;
static int32_t createVnodeModifOpStmt(SParseContext* pCxt, SNode** pOutput) {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT);
if (NULL == pStmt) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// merge according to vgId
if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
CHECK_CODE(insMergeTableDataBlocks(pCxt->pTableBlockHashObj, &pCxt->pVgDataBlocks));
}
return insBuildOutput(pCxt);
}
static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) {
STableDataBlocks* dataBuf = NULL;
CHECK_CODE(getTableMeta(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName));
CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid, sizeof(pCxt->pTableMeta->uid),
TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk),
getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL,
&pCxt->createTblReq));
CHECK_CODE(parseDataFromFileAgain(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName, dataBuf));
if (taosEOFFile(pCxt->pComCxt->csvCxt.fp)) {
CHECK_CODE(parseInsertBody(pCxt));
pCxt->pComCxt->needMultiParse = false;
return TSDB_CODE_SUCCESS;
if (pCxt->pStmtCb) {
TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT);
}
parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
// merge according to vgId
if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
CHECK_CODE(insMergeTableDataBlocks(pCxt->pTableBlockHashObj, &pCxt->pVgDataBlocks));
pStmt->pSql = pCxt->pSql;
pStmt->freeHashFunc = insDestroyBlockHashmap;
pStmt->freeArrayFunc = insDestroyBlockArrayList;
pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
pStmt->pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
pStmt->pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
pStmt->pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
pStmt->pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
if (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj || NULL == pStmt->pSubTableHashObj ||
NULL == pStmt->pTableNameHashObj || NULL == pStmt->pDbFNameHashObj) {
nodesDestroyNode((SNode*)pStmt);
return TSDB_CODE_OUT_OF_MEMORY;
}
return insBuildOutput(pCxt);
taosHashSetFreeFp(pStmt->pSubTableHashObj, destroySubTableHashElem);
*pOutput = (SNode*)pStmt;
return TSDB_CODE_SUCCESS;
}
// INSERT INTO
// tb_name
// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
// [(field1_name, ...)]
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
// [...];
int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) {
SInsertParseContext context = {
.pComCxt = pContext,
.pSql = pContext->needMultiParse ? (char*)pContext->csvCxt.pLastSqlPos : (char*)pContext->pSql,
.msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
.pTableMeta = NULL,
.createTblReq = {0},
.pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
.pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
.pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
.totalNum = 0,
.pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT),
.pStmtCb = pContext->pStmtCb,
.pMetaCache = pMetaCache,
.memElapsed = 0,
.parRowElapsed = 0};
if (pContext->pStmtCb && *pQuery) {
(*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj,
&context.pTableBlockHashObj);
if (NULL == context.pVgroupsHashObj) {
context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
}
if (NULL == context.pTableBlockHashObj) {
context.pTableBlockHashObj =
taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
}
} else {
context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
context.pTableBlockHashObj =
taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
static int32_t createInsertQuery(SParseContext* pCxt, SQuery** pOutput) {
SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj ||
NULL == context.pTableNameHashObj || NULL == context.pDbFNameHashObj || NULL == context.pOutput) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
taosHashSetFreeFp(context.pSubTableHashObj, destroySubTableHashElem);
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->haveResultSet = false;
pQuery->msgType = TDMT_VND_SUBMIT;
if (pContext->pStmtCb) {
TSDB_QUERY_SET_TYPE(context.pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT);
int32_t code = createVnodeModifOpStmt(pCxt, &pQuery->pRoot);
if (TSDB_CODE_SUCCESS == code) {
*pOutput = pQuery;
} else {
nodesDestroyNode((SNode*)pQuery);
}
return code;
}
if (NULL == *pQuery) {
*pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == *pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
nodesDestroyNode((*pQuery)->pRoot);
static int32_t checkAuthFromMetaData(const SArray* pUsers) {
if (1 != taosArrayGetSize(pUsers)) {
return TSDB_CODE_FAILED;
}
(*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE;
(*pQuery)->haveResultSet = false;
(*pQuery)->msgType = TDMT_VND_SUBMIT;
(*pQuery)->pRoot = (SNode*)context.pOutput;
SMetaRes* pRes = taosArrayGet(pUsers, 0);
if (TSDB_CODE_SUCCESS == pRes->code) {
return (*(bool*)pRes->pRes) ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED;
}
return pRes->code;
}
if (NULL == (*pQuery)->pTableList) {
(*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName));
if (NULL == (*pQuery)->pTableList) {
static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMeta) {
if (1 != taosArrayGetSize(pTables)) {
return TSDB_CODE_FAILED;
}
SMetaRes* pRes = taosArrayGet(pTables, 0);
if (TSDB_CODE_SUCCESS == pRes->code) {
*pMeta = tableMetaDup((const STableMeta*)pRes->pRes);
if (NULL == *pMeta) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return pRes->code;
}
if (NULL == (*pQuery)->pDbList) {
(*pQuery)->pDbList = taosArrayInit(taosHashGetSize(context.pDbFNameHashObj), TSDB_DB_FNAME_LEN);
if (NULL == (*pQuery)->pDbList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpStmt* pStmt, bool isStb) {
if (1 != taosArrayGetSize(pTables)) {
return TSDB_CODE_FAILED;
}
int32_t code = TSDB_CODE_SUCCESS;
if (!context.pComCxt->needMultiParse) {
code = skipInsertInto(&context.pSql, &context.msg);
if (TSDB_CODE_SUCCESS == code) {
code = parseInsertBody(&context);
}
} else {
code = parseInsertBodyAgain(&context);
SMetaRes* pRes = taosArrayGet(pTables, 0);
if (TSDB_CODE_SUCCESS != pRes->code) {
return pRes->code;
}
if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) {
SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL);
while (NULL != pTable) {
taosArrayPush((*pQuery)->pTableList, pTable);
pTable = taosHashIterate(context.pTableNameHashObj, pTable);
}
SVgroupInfo* pVg = pRes->pRes;
if (isStb) {
pStmt->pTableMeta->vgId = pVg->vgId;
}
return taosHashPut(pStmt->pVgroupsHashObj, (const char*)&pVg->vgId, sizeof(pVg->vgId), (char*)pVg,
sizeof(SVgroupInfo));
}
char* pDb = taosHashIterate(context.pDbFNameHashObj, NULL);
while (NULL != pDb) {
taosArrayPush((*pQuery)->pDbList, pDb);
pDb = taosHashIterate(context.pDbFNameHashObj, pDb);
}
static int32_t getTableSchemaFromMetaData(const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt, bool isStb) {
int32_t code = checkAuthFromMetaData(pMetaData->pUser);
if (TSDB_CODE_SUCCESS == code) {
code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta);
}
if (pContext->pStmtCb) {
context.pVgroupsHashObj = NULL;
context.pTableBlockHashObj = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb);
}
destroyInsertParseContext(&context);
return code;
}
// pSql -> (field1_value, ...) [(field1_value2, ...) ...]
static int32_t skipValuesClause(SInsertParseSyntaxCxt* pCxt) {
int32_t numOfRows = 0;
SToken sToken;
while (1) {
int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index);
if (TK_NK_LP != sToken.type) {
break;
static int32_t setVnodeModifOpStmt(SParseContext* pCxt, const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt) {
if (pCxt->pStmtCb) {
(*pCxt->pStmtCb->getExecInfoFn)(pCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj);
if (NULL == pStmt->pVgroupsHashObj) {
pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
}
if (NULL == pStmt->pTableBlockHashObj) {
pStmt->pTableBlockHashObj =
taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
}
pCxt->pSql += index;
CHECK_CODE(skipParentheses(pCxt));
++numOfRows;
TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT);
}
if (0 == numOfRows) {
return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
if (pStmt->usingTableProcessing) {
return getTableSchemaFromMetaData(pMetaData, pStmt, true);
}
return TSDB_CODE_SUCCESS;
return getTableSchemaFromMetaData(pMetaData, pStmt, false);
}
static int32_t skipTagsClause(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); }
static int32_t skipTableOptions(SInsertParseSyntaxCxt* pCxt) {
do {
int32_t index = 0;
SToken sToken;
NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index);
if (TK_TTL == sToken.type || TK_COMMENT == sToken.type) {
pCxt->pSql += index;
NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
} else {
break;
}
} while (1);
return TSDB_CODE_SUCCESS;
static int32_t initInsertQuery(SParseContext* pCxt, const SMetaData* pMetaData, SQuery** pQuery) {
if (NULL == *pQuery) {
return createInsertQuery(pCxt, pQuery);
}
return setVnodeModifOpStmt(pCxt, pMetaData, (SVnodeModifOpStmt*)(*pQuery)->pRoot);
}
// pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...)
static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) {
SToken sToken;
NEXT_TOKEN(pCxt->pSql, sToken);
if (TK_NK_LP == sToken.type) {
CHECK_CODE(skipBoundColumns(pCxt));
NEXT_TOKEN(pCxt->pSql, sToken);
static int32_t setRefreshMate(SQuery* pQuery) {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot;
SName* pTable = taosHashIterate(pStmt->pTableNameHashObj, NULL);
while (NULL != pTable) {
taosArrayPush(pQuery->pTableList, pTable);
pTable = taosHashIterate(pStmt->pTableNameHashObj, pTable);
}
if (TK_TAGS != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z);
}
// pSql -> (tag1_value, ...)
NEXT_TOKEN(pCxt->pSql, sToken);
if (TK_NK_LP != sToken.type) {
return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z);
char* pDb = taosHashIterate(pStmt->pDbFNameHashObj, NULL);
while (NULL != pDb) {
taosArrayPush(pQuery->pDbList, pDb);
pDb = taosHashIterate(pStmt->pDbFNameHashObj, pDb);
}
CHECK_CODE(skipTagsClause(pCxt));
CHECK_CODE(skipTableOptions(pCxt));
return TSDB_CODE_SUCCESS;
}
static int32_t collectTableMetaKey(SInsertParseSyntaxCxt* pCxt, bool isStable, int32_t tableNo, SToken* pTbToken) {
SName name = {0};
CHECK_CODE(insCreateSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg));
CHECK_CODE(reserveTableMetaInCacheForInsert(&name, isStable ? CATALOG_REQ_TYPE_META : CATALOG_REQ_TYPE_BOTH, tableNo,
pCxt->pMetaCache));
return TSDB_CODE_SUCCESS;
// INSERT INTO
// tb_name
// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]]
// [(field1_name, ...)]
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
// [...];
static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
int32_t code = skipInsertInto(&pStmt->pSql, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code) {
code = parseInsertBody(pCxt, pStmt);
}
return code;
}
static int32_t checkTableName(const char* pTableName, SMsgBuf* pMsgBuf) {
if (NULL != strchr(pTableName, '.')) {
return generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'");
static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
STableDataBlocks* pDataBuf = NULL;
int32_t code = getTableDataBlocks(pCxt, pStmt, &pDataBuf);
if (TSDB_CODE_SUCCESS == code) {
code = parseDataFromFileImpl(pCxt, pStmt, pDataBuf);
}
return TSDB_CODE_SUCCESS;
}
static int32_t collectAutoCreateTableMetaKey(SInsertParseSyntaxCxt* pCxt, int32_t tableNo, SToken* pTbToken) {
SName name = {0};
CHECK_CODE(insCreateSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg));
CHECK_CODE(checkTableName(name.tname, &pCxt->msg));
CHECK_CODE(reserveTableMetaInCacheForInsert(&name, CATALOG_REQ_TYPE_VGROUP, tableNo, pCxt->pMetaCache));
return TSDB_CODE_SUCCESS;
parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pStmt->totalRowsNum);
if (TSDB_CODE_SUCCESS == code) {
if (pStmt->fileProcessing) {
code = parseInsertBodyBottom(pCxt, pStmt);
} else {
code = parseInsertBody(pCxt, pStmt);
}
}
return code;
}
static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) {
bool hasData = false;
int32_t tableNo = 0;
// for each table
while (1) {
SToken sToken;
static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
int32_t code = parseInsertTableClauseBottom(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
code = parseInsertBody(pCxt, pStmt);
}
return code;
}
// pSql -> tb_name ...
NEXT_TOKEN(pCxt->pSql, sToken);
static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) {
if (pStmt->pSql == pCxt->pComCxt->pSql || NULL != pCxt->pComCxt->pStmtCb) {
return parseInsertSqlFromStart(pCxt, pStmt);
}
// no data in the sql string anymore.
if (sToken.n == 0) {
if (sToken.type && pCxt->pSql[0]) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z);
}
if (pStmt->fileProcessing) {
return parseInsertSqlFromCsv(pCxt, pStmt);
}
if (!hasData) {
return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");
}
break;
}
return parseInsertSqlFromTable(pCxt, pStmt);
}
hasData = false;
static int32_t buildInsertTableReq(SName* pName, SArray** pTables) {
*pTables = taosArrayInit(1, sizeof(SName));
if (NULL == *pTables) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SToken tbnameToken = sToken;
NEXT_TOKEN(pCxt->pSql, sToken);
taosArrayPush(*pTables, pName);
return TSDB_CODE_SUCCESS;
}
bool existedUsing = false;
// USING clause
if (TK_USING == sToken.type) {
existedUsing = true;
CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, tableNo, &tbnameToken));
NEXT_TOKEN(pCxt->pSql, sToken);
CHECK_CODE(collectTableMetaKey(pCxt, true, tableNo, &sToken));
CHECK_CODE(skipUsingClause(pCxt));
NEXT_TOKEN(pCxt->pSql, sToken);
static int32_t buildInsertDbReq(SName* pName, SArray** pDbs) {
if (NULL == *pDbs) {
*pDbs = taosArrayInit(1, sizeof(STablesReq));
if (NULL == *pDbs) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TK_NK_LP == sToken.type) {
// pSql -> field1_name, ...)
CHECK_CODE(skipBoundColumns(pCxt));
NEXT_TOKEN(pCxt->pSql, sToken);
}
STablesReq req = {0};
tNameGetFullDbName(pName, req.dbFName);
buildInsertTableReq(pName, &req.pTables);
taosArrayPush(*pDbs, &req);
if (TK_USING == sToken.type && !existedUsing) {
existedUsing = true;
CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, tableNo, &tbnameToken));
NEXT_TOKEN(pCxt->pSql, sToken);
CHECK_CODE(collectTableMetaKey(pCxt, true, tableNo, &sToken));
CHECK_CODE(skipUsingClause(pCxt));
NEXT_TOKEN(pCxt->pSql, sToken);
} else if (!existedUsing) {
CHECK_CODE(collectTableMetaKey(pCxt, false, tableNo, &tbnameToken));
}
return TSDB_CODE_SUCCESS;
}
++tableNo;
static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** pUserAuth) {
*pUserAuth = taosArrayInit(1, sizeof(SUserAuthInfo));
if (NULL == *pUserAuth) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (TK_VALUES == sToken.type) {
// pSql -> (field1_value, ...) [(field1_value2, ...) ...]
CHECK_CODE(skipValuesClause(pCxt));
hasData = true;
continue;
}
SUserAuthInfo userAuth = {.type = AUTH_TYPE_WRITE};
snprintf(userAuth.user, sizeof(userAuth.user), "%s", pUser);
tNameGetFullDbName(pName, userAuth.dbFName);
taosArrayPush(*pUserAuth, &userAuth);
// FILE csv_file_path
if (TK_FILE == sToken.type) {
// pSql -> csv_file_path
NEXT_TOKEN(pCxt->pSql, sToken);
if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) {
return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z);
}
hasData = true;
continue;
return TSDB_CODE_SUCCESS;
}
static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SCatalogReq* pCatalogReq) {
int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser);
if (TSDB_CODE_SUCCESS == code) {
if (0 == pStmt->usingTableName.type) {
code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableMeta);
} else {
code = buildInsertDbReq(&pStmt->usingTableName, &pCatalogReq->pTableMeta);
}
}
if (TSDB_CODE_SUCCESS == code) {
code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableHash);
}
return code;
}
return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z);
static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) {
if (pCxt->missCache) {
pQuery->execStage = QUERY_EXEC_STAGE_PARSE;
return buildInsertCatalogReq(pCxt, (SVnodeModifOpStmt*)pQuery->pRoot, pCatalogReq);
}
pQuery->execStage = QUERY_EXEC_STAGE_SCHEDULE;
return TSDB_CODE_SUCCESS;
}
int32_t parseInsertSyntax(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) {
SInsertParseSyntaxCxt context = {.pComCxt = pContext,
.pSql = (char*)pContext->pSql,
.msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
.pMetaCache = pMetaCache};
int32_t code = skipInsertInto(&context.pSql, &context.msg);
int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData) {
SInsertParseContext context = {
.pComCxt = pCxt,
.msg = {.buf = pCxt->pMsg, .len = pCxt->msgLen},
};
int32_t code = initInsertQuery(pCxt, pMetaData, pQuery);
if (TSDB_CODE_SUCCESS == code) {
code = parseInsertBodySyntax(&context);
code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
*pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == *pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = setNextStageInfo(&context, *pQuery, pCatalogReq);
}
if ((TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) &&
QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) {
code = setRefreshMate(*pQuery);
}
destroyBoundColumnInfo(&context.tags);
return code;
}
source/libs/parser/src/parInsertStmt.c
浏览文件 @ 8513dc10
......@@ -30,22 +30,17 @@ typedef struct SKvParam {
} SKvParam;
int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) {
SVnodeModifOpStmt* modifyNode = (SVnodeModifOpStmt*)pQuery->pRoot;
int32_t code = 0;
SInsertParseContext insertCtx = {
.pVgroupsHashObj = pVgHash,
.pTableBlockHashObj = pBlockHash,
.pOutput = (SVnodeModifOpStmt*)pQuery->pRoot,
};
SVnodeModifOpStmt* modifyNode = (SVnodeModifOpStmt*)pQuery->pRoot;
int32_t code = 0;
// merge according to vgId
if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) {
CHECK_CODE(insMergeTableDataBlocks(insertCtx.pTableBlockHashObj, &insertCtx.pVgDataBlocks));
if (taosHashGetSize(pBlockHash) > 0) {
CHECK_CODE(insMergeTableDataBlocks(pBlockHash, &modifyNode->pVgDataBlocks));
}
CHECK_CODE(insBuildOutput(&insertCtx));
CHECK_CODE(insBuildOutput(modifyNode));
insDestroyBlockArrayList(insertCtx.pVgDataBlocks);
insDestroyBlockArrayList(modifyNode->pVgDataBlocks);
return TSDB_CODE_SUCCESS;
}
......@@ -222,11 +217,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in
}
SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData);
if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, bind->num)) {
return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX");
}
return TSDB_CODE_SUCCESS;
return insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf);
}
int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
......@@ -308,10 +299,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu
pDataBlock->size += extendedRowSize * bind->num;
SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData);
if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, bind->num)) {
return buildInvalidOperationMsg(&pBuf,
"too many rows in sql, total number of rows should be less than INT32_MAX");
}
CHECK_CODE(insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf));
}
return TSDB_CODE_SUCCESS;
......
source/libs/parser/src/parInsertUtil.c
浏览文件 @ 8513dc10
......@@ -110,18 +110,17 @@ void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo* spd, col_id_t i
}
}
int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows) {
int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows, SMsgBuf* pMsg) {
pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid);
pBlocks->uid = dataBuf->pTableMeta->uid;
pBlocks->sversion = dataBuf->pTableMeta->sversion;
pBlocks->schemaLen = dataBuf->createTbReqLen;
if (pBlocks->numOfRows + numOfRows >= INT32_MAX) {
return TSDB_CODE_TSC_INVALID_OPERATION;
} else {
pBlocks->numOfRows += numOfRows;
return TSDB_CODE_SUCCESS;
return buildInvalidOperationMsg(pMsg, "too many rows in sql, total number of rows should be less than INT32_MAX");
}
pBlocks->numOfRows += numOfRows;
return TSDB_CODE_SUCCESS;
}
void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) {
......@@ -271,12 +270,8 @@ void insDestroyDataBlock(STableDataBlocks* pDataBlock) {
}
taosMemoryFreeClear(pDataBlock->pData);
// if (!pDataBlock->cloned) {
// free the refcount for metermeta
taosMemoryFreeClear(pDataBlock->pTableMeta);
destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
// }
taosMemoryFreeClear(pDataBlock);
}
......@@ -312,20 +307,6 @@ int32_t insGetDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, in
return TSDB_CODE_SUCCESS;
}
#if 0
static int32_t getRowExpandSize(STableMeta* pTableMeta) {
int32_t result = TD_ROW_HEAD_LEN - sizeof(TSKEY);
int32_t columns = getNumOfColumns(pTableMeta);
SSchema* pSchema = getTableColumnSchema(pTableMeta);
for (int32_t i = 0; i < columns; ++i) {
if (IS_VAR_DATA_TYPE((pSchema + i)->type)) {
result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY];
}
}
result += (int32_t)TD_BITMAP_BYTES(columns - 1);
return result;
}
#endif
void insDestroyBlockArrayList(SArray* pDataBlockList) {
if (pDataBlockList == NULL) {
......@@ -357,51 +338,6 @@ void insDestroyBlockHashmap(SHashObj* pDataBlockHash) {
taosHashCleanup(pDataBlockHash);
}
#if 0
// data block is disordered, sort it in ascending order
void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) {
SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData;
// size is less than the total size, since duplicated rows may be removed yet.
assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size);
if (!dataBuf->ordered) {
char* pBlockData = pBlocks->data;
// todo. qsort is unstable, if timestamp is same, should get the last one
taosSort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar);
int32_t i = 0;
int32_t j = 1;
// delete rows with timestamp conflicts
while (j < pBlocks->numOfRows) {
TSKEY ti = *(TSKEY*)(pBlockData + dataBuf->rowSize * i);
TSKEY tj = *(TSKEY*)(pBlockData + dataBuf->rowSize * j);
if (ti == tj) {
++j;
continue;
}
int32_t nextPos = (++i);
if (nextPos != j) {
memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize);
}
++j;
}
dataBuf->ordered = true;
pBlocks->numOfRows = i + 1;
dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows;
}
dataBuf->prevTS = INT64_MIN;
}
#endif
// data block is disordered, sort it in ascending order
static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) {
SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData;
......@@ -994,24 +930,24 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) {
}
}
int32_t insBuildOutput(SInsertParseContext* pCxt) {
size_t numOfVg = taosArrayGetSize(pCxt->pVgDataBlocks);
pCxt->pOutput->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES);
if (NULL == pCxt->pOutput->pDataBlocks) {
int32_t insBuildOutput(SVnodeModifOpStmt* pStmt) {
size_t numOfVg = taosArrayGetSize(pStmt->pVgDataBlocks);
pStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES);
if (NULL == pStmt->pDataBlocks) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
for (size_t i = 0; i < numOfVg; ++i) {
STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i);
STableDataBlocks* src = taosArrayGetP(pStmt->pVgDataBlocks, i);
SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
if (NULL == dst) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
taosHashGetDup(pStmt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
dst->numOfTables = src->numOfTables;
dst->size = src->size;
TSWAP(dst->pData, src->pData);
buildMsgHeader(src, dst);
taosArrayPush(pCxt->pOutput->pDataBlocks, &dst);
taosArrayPush(pStmt->pDataBlocks, &dst);
}
return TSDB_CODE_SUCCESS;
}
source/libs/parser/src/parUtil.c
浏览文件 @ 8513dc10
......@@ -612,62 +612,7 @@ static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) {
return TSDB_CODE_SUCCESS;
}
static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCache* pMetaCache,
SCatalogReq* pCatalogReq) {
int32_t ndbs = taosHashGetSize(pMetaCache->pInsertTables);
pCatalogReq->pTableMeta = taosArrayInit(ndbs, sizeof(STablesReq));
if (NULL == pCatalogReq->pTableMeta) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCatalogReq->pTableHash = taosArrayInit(ndbs, sizeof(STablesReq));
if (NULL == pCatalogReq->pTableHash) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCatalogReq->pUser = taosArrayInit(ndbs, sizeof(SUserAuthInfo));
if (NULL == pCatalogReq->pUser) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pTableMetaPos = taosArrayInit(pMetaCache->sqlTableNum, sizeof(int32_t));
pCxt->pTableVgroupPos = taosArrayInit(pMetaCache->sqlTableNum, sizeof(int32_t));
int32_t metaReqNo = 0;
int32_t vgroupReqNo = 0;
SInsertTablesMetaReq* p = taosHashIterate(pMetaCache->pInsertTables, NULL);
while (NULL != p) {
STablesReq req = {0};
strcpy(req.dbFName, p->dbFName);
TSWAP(req.pTables, p->pTableMetaReq);
taosArrayPush(pCatalogReq->pTableMeta, &req);
req.pTables = NULL;
TSWAP(req.pTables, p->pTableVgroupReq);
taosArrayPush(pCatalogReq->pTableHash, &req);
int32_t ntables = taosArrayGetSize(p->pTableMetaPos);
for (int32_t i = 0; i < ntables; ++i) {
taosArrayInsert(pCxt->pTableMetaPos, *(int32_t*)taosArrayGet(p->pTableMetaPos, i), &metaReqNo);
++metaReqNo;
}
ntables = taosArrayGetSize(p->pTableVgroupPos);
for (int32_t i = 0; i < ntables; ++i) {
taosArrayInsert(pCxt->pTableVgroupPos, *(int32_t*)taosArrayGet(p->pTableVgroupPos, i), &vgroupReqNo);
++vgroupReqNo;
}
SUserAuthInfo auth = {0};
snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser);
snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName);
auth.type = AUTH_TYPE_WRITE;
taosArrayPush(pCatalogReq->pUser, &auth);
p = taosHashIterate(pMetaCache->pInsertTables, p);
}
return TSDB_CODE_SUCCESS;
}
int32_t buildCatalogReqForQuery(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) {
int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) {
int32_t code = buildTableReqFromDb(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta);
if (TSDB_CODE_SUCCESS == code) {
code = buildDbReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup);
......@@ -697,13 +642,6 @@ int32_t buildCatalogReqForQuery(const SParseMetaCache* pMetaCache, SCatalogReq*
return code;
}
int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) {
if (NULL != pMetaCache->pInsertTables) {
return buildCatalogReqForInsert(pCxt, pMetaCache, pCatalogReq);
}
return buildCatalogReqForQuery(pMetaCache, pCatalogReq);
}
static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj** pHash) {
if (NULL == *pHash) {
*pHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
......@@ -791,8 +729,7 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas
return TSDB_CODE_SUCCESS;
}
int32_t putMetaDataToCacheForQuery(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData,
SParseMetaCache* pMetaCache) {
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) {
int32_t code = putDbTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta);
if (TSDB_CODE_SUCCESS == code) {
code = putDbDataToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, &pMetaCache->pDbVgroup);
......@@ -822,30 +759,6 @@ int32_t putMetaDataToCacheForQuery(const SCatalogReq* pCatalogReq, const SMetaDa
return code;
}
int32_t putMetaDataToCacheForInsert(const SMetaData* pMetaData, SParseMetaCache* pMetaCache) {
int32_t ndbs = taosArrayGetSize(pMetaData->pUser);
for (int32_t i = 0; i < ndbs; ++i) {
SMetaRes* pRes = taosArrayGet(pMetaData->pUser, i);
if (TSDB_CODE_SUCCESS != pRes->code) {
return pRes->code;
}
if (!(*(bool*)pRes->pRes)) {
return TSDB_CODE_PAR_PERMISSION_DENIED;
}
}
pMetaCache->pTableMetaData = pMetaData->pTableMeta;
pMetaCache->pTableVgroupData = pMetaData->pTableHash;
return TSDB_CODE_SUCCESS;
}
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache,
bool insertValuesStmt) {
if (insertValuesStmt) {
return putMetaDataToCacheForInsert(pMetaData, pMetaCache);
}
return putMetaDataToCacheForQuery(pCatalogReq, pMetaData, pMetaCache);
}
static int32_t reserveTableReqInCacheImpl(const char* pTbFName, int32_t len, SHashObj** pTables) {
if (NULL == *pTables) {
*pTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
......@@ -1146,82 +1059,6 @@ int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes) {
return TSDB_CODE_SUCCESS;
}
static int32_t reserveTableReqInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo,
SInsertTablesMetaReq* pReq) {
switch (reqType) {
case CATALOG_REQ_TYPE_META:
taosArrayPush(pReq->pTableMetaReq, pName);
taosArrayPush(pReq->pTableMetaPos, &tableNo);
break;
case CATALOG_REQ_TYPE_VGROUP:
taosArrayPush(pReq->pTableVgroupReq, pName);
taosArrayPush(pReq->pTableVgroupPos, &tableNo);
break;
case CATALOG_REQ_TYPE_BOTH:
taosArrayPush(pReq->pTableMetaReq, pName);
taosArrayPush(pReq->pTableMetaPos, &tableNo);
taosArrayPush(pReq->pTableVgroupReq, pName);
taosArrayPush(pReq->pTableVgroupPos, &tableNo);
break;
default:
break;
}
return TSDB_CODE_SUCCESS;
}
static int32_t reserveTableReqInDbCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo,
SHashObj* pDbs) {
SInsertTablesMetaReq req = {.pTableMetaReq = taosArrayInit(4, sizeof(SName)),
.pTableMetaPos = taosArrayInit(4, sizeof(int32_t)),
.pTableVgroupReq = taosArrayInit(4, sizeof(SName)),
.pTableVgroupPos = taosArrayInit(4, sizeof(int32_t))};
tNameGetFullDbName(pName, req.dbFName);
int32_t code = reserveTableReqInCacheForInsert(pName, reqType, tableNo, &req);
if (TSDB_CODE_SUCCESS == code) {
code = taosHashPut(pDbs, pName->dbname, strlen(pName->dbname), &req, sizeof(SInsertTablesMetaReq));
}
return code;
}
int32_t reserveTableMetaInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo,
SParseMetaCache* pMetaCache) {
if (NULL == pMetaCache->pInsertTables) {
pMetaCache->pInsertTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (NULL == pMetaCache->pInsertTables) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
pMetaCache->sqlTableNum = tableNo;
SInsertTablesMetaReq* pReq = taosHashGet(pMetaCache->pInsertTables, pName->dbname, strlen(pName->dbname));
if (NULL == pReq) {
return reserveTableReqInDbCacheForInsert(pName, reqType, tableNo, pMetaCache->pInsertTables);
}
return reserveTableReqInCacheForInsert(pName, reqType, tableNo, pReq);
}
int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* pMetaCache, int32_t tableNo,
STableMeta** pMeta) {
int32_t reqIndex = *(int32_t*)taosArrayGet(pTableMetaPos, tableNo);
SMetaRes* pRes = taosArrayGet(pMetaCache->pTableMetaData, reqIndex);
if (TSDB_CODE_SUCCESS == pRes->code) {
*pMeta = tableMetaDup((const STableMeta*)pRes->pRes);
if (NULL == *pMeta) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return pRes->code;
}
int32_t getTableVgroupFromCacheForInsert(SArray* pTableVgroupPos, SParseMetaCache* pMetaCache, int32_t tableNo,
SVgroupInfo* pVgroup) {
int32_t reqIndex = *(int32_t*)taosArrayGet(pTableVgroupPos, tableNo);
SMetaRes* pRes = taosArrayGet(pMetaCache->pTableVgroupData, reqIndex);
if (TSDB_CODE_SUCCESS == pRes->code) {
memcpy(pVgroup, pRes->pRes, sizeof(SVgroupInfo));
}
return pRes->code;
}
void destoryParseTablesMetaReqHash(SHashObj* pHash) {
SParseTablesMetaReq* p = taosHashIterate(pHash, NULL);
while (NULL != p) {
......@@ -1239,16 +1076,6 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) {
taosHashCleanup(pMetaCache->pTableMeta);
taosHashCleanup(pMetaCache->pTableVgroup);
}
SInsertTablesMetaReq* p = taosHashIterate(pMetaCache->pInsertTables, NULL);
while (NULL != p) {
taosArrayDestroy(p->pTableMetaPos);
taosArrayDestroy(p->pTableMetaReq);
taosArrayDestroy(p->pTableVgroupPos);
taosArrayDestroy(p->pTableVgroupReq);
p = taosHashIterate(pMetaCache->pInsertTables, p);
}
taosHashCleanup(pMetaCache->pInsertTables);
taosHashCleanup(pMetaCache->pDbVgroup);
taosHashCleanup(pMetaCache->pDbCfg);
taosHashCleanup(pMetaCache->pDbInfo);
......
source/libs/parser/src/parser.c
浏览文件 @ 8513dc10
......@@ -167,7 +167,7 @@ static void rewriteExprAlias(SNode* pRoot) {
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) {
int32_t code = TSDB_CODE_SUCCESS;
if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) {
code = parseInsertSql(pCxt, pQuery, NULL);
code = parseInsertSql(pCxt, pQuery, NULL, NULL);
} else {
code = parseSqlIntoAst(pCxt, pQuery);
}
......@@ -175,21 +175,26 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) {
return code;
}
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
static int32_t parseQuerySyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
SParseMetaCache metaCache = {0};
int32_t code = nodesAcquireAllocator(pCxt->allocatorId);
int32_t code = parseSqlSyntax(pCxt, pQuery, &metaCache);
if (TSDB_CODE_SUCCESS == code) {
code = buildCatalogReq(&metaCache, pCatalogReq);
}
destoryParseMetaCache(&metaCache, true);
return code;
}
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
int32_t code = nodesAcquireAllocator(pCxt->allocatorId);
if (TSDB_CODE_SUCCESS == code) {
if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) {
code = parseInsertSyntax(pCxt, pQuery, &metaCache);
code = parseInsertSql(pCxt, pQuery, pCatalogReq, NULL);
} else {
code = parseSqlSyntax(pCxt, pQuery, &metaCache);
code = parseQuerySyntax(pCxt, pQuery, pCatalogReq);
}
}
if (TSDB_CODE_SUCCESS == code) {
code = buildCatalogReq(pCxt, &metaCache, pCatalogReq);
}
nodesReleaseAllocator(pCxt->allocatorId);
destoryParseMetaCache(&metaCache, true);
terrno = code;
return code;
}
......@@ -199,14 +204,10 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
SParseMetaCache metaCache = {0};
int32_t code = nodesAcquireAllocator(pCxt->allocatorId);
if (TSDB_CODE_SUCCESS == code) {
code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot);
code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache);
}
if (TSDB_CODE_SUCCESS == code) {
if (NULL == pQuery->pRoot) {
code = parseInsertSql(pCxt, &pQuery, &metaCache);
} else {
code = analyseSemantic(pCxt, pQuery, &metaCache);
}
code = analyseSemantic(pCxt, pQuery, &metaCache);
}
nodesReleaseAllocator(pCxt->allocatorId);
destoryParseMetaCache(&metaCache, false);
......@@ -214,6 +215,11 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
return code;
}
int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData,
SQuery* pQuery) {
return parseInsertSql(pCxt, &pQuery, pCatalogReq, pMetaData);
}
void qDestroyParseContext(SParseContext* pCxt) {
if (NULL == pCxt) {
return;
......
source/libs/parser/test/mockCatalog.cpp
浏览文件 @ 8513dc10
......@@ -228,11 +228,23 @@ int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, SRequestConnInfo* pConn
return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta);
}
int32_t __catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta) {
return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta, true);
}
int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName,
SVgroupInfo* vgInfo) {
return g_mockCatalogService->catalogGetTableHashVgroup(pTableName, vgInfo);
}
int32_t __catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
SVgroupInfo* pVgroup, bool* exists) {
int32_t code = g_mockCatalogService->catalogGetTableHashVgroup(pTableName, pVgroup, true);
*exists = 0 != pVgroup->vgId;
return code;
}
int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
SArray** pVgList) {
return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList);
......@@ -289,8 +301,11 @@ void initMetaDataEnv() {
static Stub stub;
stub.set(catalogGetHandle, __catalogGetHandle);
stub.set(catalogGetTableMeta, __catalogGetTableMeta);
stub.set(catalogGetCachedTableMeta, __catalogGetCachedTableMeta);
stub.set(catalogGetSTableMeta, __catalogGetTableMeta);
stub.set(catalogGetCachedSTableMeta, __catalogGetCachedTableMeta);
stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup);
stub.set(catalogGetCachedTableHashVgroup, __catalogGetCachedTableHashVgroup);
stub.set(catalogGetTableDistVgInfo, __catalogGetTableDistVgInfo);
stub.set(catalogGetDBVgVersion, __catalogGetDBVgVersion);
stub.set(catalogGetDBVgList, __catalogGetDBVgList);
......
source/libs/parser/test/mockCatalogService.cpp
浏览文件 @ 8513dc10
......@@ -91,7 +91,7 @@ class MockCatalogServiceImpl {
public:
static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]);
MockCatalogServiceImpl() : id_(1) {}
MockCatalogServiceImpl() : id_(1), havaCache_(true) {}
~MockCatalogServiceImpl() {
for (auto& cfg : dbCfg_) {
......@@ -106,7 +106,11 @@ class MockCatalogServiceImpl {
int32_t catalogGetHandle() const { return 0; }
int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const {
int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, bool onlyCache = false) const {
if (onlyCache && !havaCache_) {
return TSDB_CODE_SUCCESS;
}
std::unique_ptr<STableMeta> table;
char db[TSDB_DB_NAME_LEN] = {0};
......@@ -121,7 +125,12 @@ class MockCatalogServiceImpl {
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, bool onlyCache = false) const {
if (onlyCache && !havaCache_) {
vgInfo->vgId = 0;
return TSDB_CODE_SUCCESS;
}
vgInfo->vgId = 1;
return TSDB_CODE_SUCCESS;
}
......@@ -618,6 +627,7 @@ class MockCatalogServiceImpl {
IndexMetaCache index_;
DnodeCache dnode_;
DbCfgCache dbCfg_;
bool havaCache_;
};
MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {}
......@@ -651,12 +661,14 @@ void MockCatalogService::createDatabase(const std::string& db, bool rollup, int8
impl_->createDatabase(db, rollup, cacheLast);
}
int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const {
return impl_->catalogGetTableMeta(pTableName, pTableMeta);
int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta,
bool onlyCache) const {
return impl_->catalogGetTableMeta(pTableName, pTableMeta, onlyCache);
}
int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
return impl_->catalogGetTableHashVgroup(pTableName, vgInfo);
int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo,
bool onlyCache) const {
return impl_->catalogGetTableHashVgroup(pTableName, vgInfo, onlyCache);
}
int32_t MockCatalogService::catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const {
......
source/libs/parser/test/mockCatalogService.h
浏览文件 @ 8513dc10
......@@ -67,8 +67,8 @@ class MockCatalogService {
void createDnode(int32_t dnodeId, const std::string& host, int16_t port);
void createDatabase(const std::string& db, bool rollup = false, int8_t cacheLast = 0);
int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const;
int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const;
int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, bool onlyCache = false) const;
int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, bool onlyCache = false) const;
int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const;
int32_t catalogGetDBVgList(const char* pDbFName, SArray** pVgList) const;
int32_t catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pDbCfg) const;
......
source/libs/parser/test/parTestUtil.cpp
浏览文件 @ 8513dc10
......@@ -233,16 +233,15 @@ class ParserTestBaseImpl {
}
void doBuildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) {
DO_WITH_THROW(buildCatalogReq, pCxt, pMetaCache, pCatalogReq);
DO_WITH_THROW(buildCatalogReq, pMetaCache, pCatalogReq);
}
void doGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) {
DO_WITH_THROW(g_mockCatalogService->catalogGetAllMeta, pCatalogReq, pMetaData);
}
void doPutMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache,
bool isInsertValues) {
DO_WITH_THROW(putMetaDataToCache, pCatalogReq, pMetaData, pMetaCache, isInsertValues);
void doPutMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) {
DO_WITH_THROW(putMetaDataToCache, pCatalogReq, pMetaData, pMetaCache);
}
void doAuthenticate(SParseContext* pCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) {
......@@ -280,15 +279,14 @@ class ParserTestBaseImpl {
res_.calcConstAst_ = toString(pQuery->pRoot);
}
void doParseInsertSql(SParseContext* pCxt, SQuery** pQuery, SParseMetaCache* pMetaCache) {
DO_WITH_THROW(parseInsertSql, pCxt, pQuery, pMetaCache);
void doParseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData) {
DO_WITH_THROW(parseInsertSql, pCxt, pQuery, pCatalogReq, pMetaData);
ASSERT_NE(*pQuery, nullptr);
res_.parsedAst_ = toString((*pQuery)->pRoot);
}
void doParseInsertSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCache* pMetaCache) {
DO_WITH_THROW(parseInsertSyntax, pCxt, pQuery, pMetaCache);
ASSERT_NE(*pQuery, nullptr);
void doContinueParseSql(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SQuery* pQuery) {
DO_WITH_THROW(qContinueParseSql, pCxt, pCatalogReq, pMetaData, pQuery);
}
string toString(const SNode* pRoot) {
......@@ -310,7 +308,7 @@ class ParserTestBaseImpl {
if (qIsInsertValuesSql(cxt.pSql, cxt.sqlLen)) {
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
doParseInsertSql(&cxt, query.get(), nullptr);
doParseInsertSql(&cxt, query.get(), nullptr, nullptr);
} else {
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
doParse(&cxt, query.get());
......@@ -356,61 +354,102 @@ class ParserTestBaseImpl {
}
}
void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) {
reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL);
try {
unique_ptr<SParseContext, function<void(SParseContext*)> > cxt(new SParseContext(), destoryParseContext);
setParseContext(sql, cxt.get(), true);
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
bool request = true;
unique_ptr<SParseMetaCache, function<void(SParseMetaCache*)> > metaCache(
new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request)));
bool isInsertValues = qIsInsertValuesSql(cxt->pSql, cxt->sqlLen);
if (isInsertValues) {
doParseInsertSyntax(cxt.get(), query.get(), metaCache.get());
} else {
doParse(cxt.get(), query.get());
doCollectMetaKey(cxt.get(), *(query.get()), metaCache.get());
void runQueryAsyncInternalFuncs(SParseContext* pParCxt) {
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
bool request = true;
unique_ptr<SParseMetaCache, function<void(SParseMetaCache*)> > metaCache(
new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request)));
doParse(pParCxt, query.get());
doCollectMetaKey(pParCxt, *(query.get()), metaCache.get());
SQuery* pQuery = *(query.get());
unique_ptr<SCatalogReq, void (*)(SCatalogReq*)> catalogReq(new SCatalogReq(),
MockCatalogService::destoryCatalogReq);
doBuildCatalogReq(pParCxt, metaCache.get(), catalogReq.get());
string err;
thread t1([&]() {
try {
unique_ptr<SMetaData, void (*)(SMetaData*)> metaData(new SMetaData(), MockCatalogService::destoryMetaData);
doGetAllMeta(catalogReq.get(), metaData.get());
metaCache.reset(new SParseMetaCache());
request = false;
doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get());
doAuthenticate(pParCxt, pQuery, metaCache.get());
doTranslate(pParCxt, pQuery, metaCache.get());
doCalculateConstant(pParCxt, pQuery);
} catch (const TerminateFlag& e) {
// success and terminate
} catch (const runtime_error& e) {
err = e.what();
} catch (...) {
err = "unknown error";
}
});
SQuery* pQuery = *(query.get());
unique_ptr<SCatalogReq, void (*)(SCatalogReq*)> catalogReq(new SCatalogReq(),
MockCatalogService::destoryCatalogReq);
doBuildCatalogReq(cxt.get(), metaCache.get(), catalogReq.get());
t1.join();
if (!err.empty()) {
throw runtime_error(err);
}
}
string err;
thread t1([&]() {
try {
unique_ptr<SMetaData, void (*)(SMetaData*)> metaData(new SMetaData(), MockCatalogService::destoryMetaData);
doGetAllMeta(catalogReq.get(), metaData.get());
void runInsertAsyncInternalFuncsImpl(SParseContext* pParCxt, SQuery** pQuery, SCatalogReq* pCatalogReq,
SMetaData* pMetaData) {
doParseInsertSql(pParCxt, pQuery, pCatalogReq, pMetaData);
metaCache.reset(new SParseMetaCache());
request = false;
doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get(), isInsertValues);
if (QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) {
return;
}
if (isInsertValues) {
doParseInsertSql(cxt.get(), query.get(), metaCache.get());
} else {
doAuthenticate(cxt.get(), pQuery, metaCache.get());
string err;
thread t1([&]() {
try {
doGetAllMeta(pCatalogReq, pMetaData);
doTranslate(cxt.get(), pQuery, metaCache.get());
doParseInsertSql(pParCxt, pQuery, pCatalogReq, pMetaData);
doCalculateConstant(cxt.get(), pQuery);
}
} catch (const TerminateFlag& e) {
// success and terminate
} catch (const runtime_error& e) {
err = e.what();
} catch (...) {
err = "unknown error";
if (QUERY_EXEC_STAGE_SCHEDULE != (*pQuery)->execStage) {
runInsertAsyncInternalFuncsImpl(pParCxt, pQuery, pCatalogReq, pMetaData);
}
});
} catch (const TerminateFlag& e) {
// success and terminate
} catch (const runtime_error& e) {
err = e.what();
} catch (...) {
err = "unknown error";
}
});
t1.join();
if (!err.empty()) {
throw runtime_error(err);
}
}
void runInsertAsyncInternalFuncs(SParseContext* pParCxt) {
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
unique_ptr<SCatalogReq, void (*)(SCatalogReq*)> catalogReq(new SCatalogReq(),
MockCatalogService::destoryCatalogReq);
unique_ptr<SMetaData, void (*)(SMetaData*)> metaData(new SMetaData(), MockCatalogService::destoryMetaData);
runInsertAsyncInternalFuncsImpl(pParCxt, query.get(), catalogReq.get(), metaData.get());
}
void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) {
reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL);
try {
unique_ptr<SParseContext, function<void(SParseContext*)> > cxt(new SParseContext(), destoryParseContext);
setParseContext(sql, cxt.get(), true);
t1.join();
if (!err.empty()) {
throw runtime_error(err);
bool isInsertValues = qIsInsertValuesSql(cxt->pSql, cxt->sqlLen);
if (isInsertValues) {
runInsertAsyncInternalFuncs(cxt.get());
} else {
runQueryAsyncInternalFuncs(cxt.get());
}
if (g_dump) {
......@@ -437,25 +476,39 @@ class ParserTestBaseImpl {
doParseSqlSyntax(cxt.get(), query.get(), catalogReq.get());
SQuery* pQuery = *(query.get());
string err;
thread t1([&]() {
try {
unique_ptr<SMetaData, void (*)(SMetaData*)> metaData(new SMetaData(), MockCatalogService::destoryMetaData);
doGetAllMeta(catalogReq.get(), metaData.get());
doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery);
} catch (const TerminateFlag& e) {
// success and terminate
} catch (const runtime_error& e) {
err = e.what();
} catch (...) {
err = "unknown error";
switch (pQuery->execStage) {
case QUERY_EXEC_STAGE_PARSE:
case QUERY_EXEC_STAGE_ANALYSE: {
string err;
thread t1([&]() {
try {
unique_ptr<SMetaData, void (*)(SMetaData*)> metaData(new SMetaData(),
MockCatalogService::destoryMetaData);
doGetAllMeta(catalogReq.get(), metaData.get());
if (QUERY_EXEC_STAGE_PARSE == pQuery->execStage) {
doContinueParseSql(cxt.get(), catalogReq.get(), metaData.get(), pQuery);
} else {
doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery);
}
} catch (const TerminateFlag& e) {
// success and terminate
} catch (const runtime_error& e) {
err = e.what();
} catch (...) {
err = "unknown error";
}
});
t1.join();
if (!err.empty()) {
throw runtime_error(err);
}
break;
}
});
t1.join();
if (!err.empty()) {
throw runtime_error(err);
case QUERY_EXEC_STAGE_SCHEDULE:
break;
default:
break;
}
if (g_dump) {
......
source/libs/scheduler/src/schRemote.c
浏览文件 @ 8513dc10
......@@ -1134,7 +1134,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
#if 1
#if 0
SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)};
code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL));
msg = NULL;
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或