Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 eb016e01 编写于 作者: H Hongze Cheng
浏览文件
操作

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/data_format

上级 038e9558 426200a7
显示空白变更内容
内联 并排
Showing with 3011 addition and 1422 deletion
example/src/tmq.c
浏览文件 @ eb016e01
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
static int running = 1; static int running = 1;
static void msg_process(TAOS_RES* msg) { static void msg_process(TAOS_RES* msg) {
char buf[1024]; char buf[1024];
memset(buf, 0, 1024); /*memset(buf, 0, 1024);*/
printf("topic: %s\n", tmq_get_topic_name(msg)); printf("topic: %s\n", tmq_get_topic_name(msg));
printf("vg: %d\n", tmq_get_vgroup_id(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg));
while (1) { while (1) {
......
include/common/tcommon.h
浏览文件 @ eb016e01
...@@ -48,6 +48,7 @@ enum { ...@@ -48,6 +48,7 @@ enum {
typedef enum EStreamType { typedef enum EStreamType {
STREAM_NORMAL = 1, STREAM_NORMAL = 1,
STREAM_INVERT, STREAM_INVERT,
STREAM_REPROCESS,
STREAM_INVALID, STREAM_INVALID,
} EStreamType; } EStreamType;
......
include/common/ttypes.h
浏览文件 @ eb016e01
...@@ -50,6 +50,7 @@ typedef struct { ...@@ -50,6 +50,7 @@ typedef struct {
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) #define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
......
include/libs/function/function.h
浏览文件 @ eb016e01
...@@ -126,7 +126,7 @@ enum { ...@@ -126,7 +126,7 @@ enum {
enum { enum {
MAIN_SCAN = 0x0u, MAIN_SCAN = 0x0u,
REVERSE_SCAN = 0x1u, REVERSE_SCAN = 0x1u, // todo remove it
REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan
MERGE_STAGE = 0x20u, MERGE_STAGE = 0x20u,
}; };
...@@ -222,13 +222,6 @@ enum { ...@@ -222,13 +222,6 @@ enum {
typedef struct tExprNode { typedef struct tExprNode {
int32_t nodeType; int32_t nodeType;
union { union {
struct {
int32_t optr; // binary operator
void *info; // support filter operation on this expression only available for leaf node
struct tExprNode *pLeft; // left child pointer
struct tExprNode *pRight; // right child pointer
} _node;
SSchema *pSchema;// column node SSchema *pSchema;// column node
struct SVariant *pVal; // value node struct SVariant *pVal; // value node
...@@ -237,12 +230,6 @@ typedef struct tExprNode { ...@@ -237,12 +230,6 @@ typedef struct tExprNode {
int32_t functionId; int32_t functionId;
int32_t num; int32_t num;
struct SFunctionNode *pFunctNode; struct SFunctionNode *pFunctNode;
// Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the
// calculation instead.
// E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes.
// The concat function, concat(col1, col2), is a binary scalar
// operator and is kept in the attribute of _node.
struct tExprNode **pChild;
} _function; } _function;
struct { struct {
...@@ -273,6 +260,7 @@ typedef struct SAggFunctionInfo { ...@@ -273,6 +260,7 @@ typedef struct SAggFunctionInfo {
struct SScalarParam { struct SScalarParam {
SColumnInfoData *columnData; SColumnInfoData *columnData;
SHashObj *pHashFilter; SHashObj *pHashFilter;
void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value
int32_t numOfRows; int32_t numOfRows;
}; };
...@@ -281,10 +269,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI ...@@ -281,10 +269,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId); bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId);
tExprNode* exprTreeFromBinary(const void* data, size_t size);
tExprNode* exprdup(tExprNode* pTree);
void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num);
void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell);
int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock); int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock);
......
include/libs/nodes/plannodes.h
浏览文件 @ eb016e01
...@@ -193,7 +193,6 @@ typedef struct SScanPhysiNode { ...@@ -193,7 +193,6 @@ typedef struct SScanPhysiNode {
} SScanPhysiNode; } SScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode;
typedef SScanPhysiNode SStreamScanPhysiNode;
typedef struct SSystemTableScanPhysiNode { typedef struct SSystemTableScanPhysiNode {
SScanPhysiNode scan; SScanPhysiNode scan;
...@@ -217,6 +216,7 @@ typedef struct STableScanPhysiNode { ...@@ -217,6 +216,7 @@ typedef struct STableScanPhysiNode {
} STableScanPhysiNode; } STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode;
typedef STableScanPhysiNode SStreamScanPhysiNode;
typedef struct SProjectPhysiNode { typedef struct SProjectPhysiNode {
SPhysiNode node; SPhysiNode node;
......
include/libs/nodes/querynodes.h
浏览文件 @ eb016e01
...@@ -324,6 +324,7 @@ typedef struct SQuery { ...@@ -324,6 +324,7 @@ typedef struct SQuery {
SArray* pTableList; SArray* pTableList;
bool showRewrite; bool showRewrite;
int32_t placeholderNum; int32_t placeholderNum;
SArray* pPlaceholderValues;
} SQuery; } SQuery;
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
......
include/libs/parser/parser.h
浏览文件 @ eb016e01
...@@ -47,7 +47,7 @@ typedef struct SParseContext { ...@@ -47,7 +47,7 @@ typedef struct SParseContext {
bool isSuperUser; bool isSuperUser;
} SParseContext; } SParseContext;
int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
bool isInsertSql(const char* pStr, size_t length); bool isInsertSql(const char* pStr, size_t length);
void qDestroyQuery(SQuery* pQueryNode); void qDestroyQuery(SQuery* pQueryNode);
...@@ -60,8 +60,10 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); ...@@ -60,8 +60,10 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc);
void qFreeStmtDataBlock(void* pDataBlock); void qFreeStmtDataBlock(void* pDataBlock);
int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId);
void qDestroyStmtDataBlock(void* pBlock); void qDestroyStmtDataBlock(void* pBlock);
STableMeta *qGetTableMetaInDataBlock(void* pDataBlock); STableMeta* qGetTableMetaInDataBlock(void* pDataBlock);
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId);
int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery);
int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
int32_t rowNum); int32_t rowNum);
...@@ -75,7 +77,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* ...@@ -75,7 +77,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
void* smlInitHandle(SQuery* pQuery); void* smlInitHandle(SQuery* pQuery);
void smlDestroyHandle(void* pHandle); void smlDestroyHandle(void* pHandle);
int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* colsSchema, SArray* cols, bool format, int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format,
STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen); STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen);
int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
......
source/client/inc/clientStmt.h
浏览文件 @ eb016e01
...@@ -71,6 +71,7 @@ typedef struct SStmtBindInfo { ...@@ -71,6 +71,7 @@ typedef struct SStmtBindInfo {
typedef struct SStmtExecInfo { typedef struct SStmtExecInfo {
int32_t affectedRows; int32_t affectedRows;
bool emptyRes;
SRequestObj* pRequest; SRequestObj* pRequest;
SHashObj* pVgHash; SHashObj* pVgHash;
SHashObj* pBlockHash; SHashObj* pBlockHash;
......
source/client/src/clientImpl.c
浏览文件 @ eb016e01
...@@ -180,7 +180,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC ...@@ -180,7 +180,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
return code; return code;
} }
code = qParseQuerySql(&cxt, pQuery); code = qParseSql(&cxt, pQuery);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if ((*pQuery)->haveResultSet) { if ((*pQuery)->haveResultSet) {
setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
......
source/client/src/clientMain.c
浏览文件 @ eb016e01
...@@ -303,6 +303,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) ...@@ -303,6 +303,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
break; break;
} }
} }
str[len] = 0;
return len; return len;
} }
...@@ -567,7 +568,7 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param ...@@ -567,7 +568,7 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param
// todo directly call fp // todo directly call fp
} }
taos_query_l(taos, sql, (int32_t) strlen(sql)); taos_query_l(taos, sql, (int32_t)strlen(sql));
} }
void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
......
source/client/src/clientSml.c
浏览文件 @ eb016e01
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
#include "catalog.h" #include "catalog.h"
#include "clientInt.h" #include "clientInt.h"
#include "tname.h" #include "tname.h"
#include "cJSON.h"
//================================================================================================= //=================================================================================================
#define SPACE ' ' #define SPACE ' '
...@@ -25,6 +26,22 @@ ...@@ -25,6 +26,22 @@
#define SLASH '\\' #define SLASH '\\'
#define tsMaxSQLStringLen (1024*1024) #define tsMaxSQLStringLen (1024*1024)
#define OTD_MAX_FIELDS_NUM 2
#define OTD_JSON_SUB_FIELDS_NUM 2
#define OTD_JSON_FIELDS_NUM 4
#define OTD_TIMESTAMP_COLUMN_NAME "ts"
#define OTD_METRIC_VALUE_COLUMN_NAME "value"
#define TS "_ts"
#define TS_LEN 3
#define TAG "_tagNone"
#define TAG_LEN 8
#define VALUE "value"
#define VALUE_LEN 5
#define BINARY_ADD_LEN 2 // "binary" 2 means " "
#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" "
//================================================================================================= //=================================================================================================
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
...@@ -70,17 +87,15 @@ typedef struct { ...@@ -70,17 +87,15 @@ typedef struct {
typedef struct { typedef struct {
const char *sTableName; // super table name const char *sTableName; // super table name
uint8_t sTableNameLen; int32_t sTableNameLen;
char childTableName[TSDB_TABLE_NAME_LEN]; char childTableName[TSDB_TABLE_NAME_LEN];
uint64_t uid; uint64_t uid;
SArray *tags; SArray *tags;
// colsFormat store cols formated, for quick parse, if info->formatData is true // if info->formatData is true, elements are SArray<SSmlKv*>.
SArray *colsFormat; // elements are SArray<SSmlKv*> // if info->formatData is false, elements are SHashObj<cols key string, SSmlKv*> for find by key quickly
SArray *cols;
// cols store cols un formated
SArray *cols; // elements are SHashObj<cols key string, SSmlKv*> for find by key quickly
} SSmlTableInfo; } SSmlTableInfo;
typedef struct { typedef struct {
...@@ -114,11 +129,11 @@ typedef struct { ...@@ -114,11 +129,11 @@ typedef struct {
} SSmlCostInfo; } SSmlCostInfo;
typedef struct { typedef struct {
uint64_t id; int64_t id;
SMLProtocolType protocol; SMLProtocolType protocol;
int8_t precision; int8_t precision;
bool dataFormat; // true means that the name, number and order of keys in each line are the same bool dataFormat; // true means that the name, number and order of keys in each line are the same(only for influx protocol)
SHashObj *childTables; SHashObj *childTables;
SHashObj *superTables; SHashObj *superTables;
...@@ -134,16 +149,12 @@ typedef struct { ...@@ -134,16 +149,12 @@ typedef struct {
int32_t affectedRows; int32_t affectedRows;
SSmlMsgBuf msgBuf; SSmlMsgBuf msgBuf;
SHashObj *dumplicateKey; // for dumplicate key SHashObj *dumplicateKey; // for dumplicate key
SArray *colsContainer; // for cols parse, if is dataFormat == false SArray *colsContainer; // for cols parse, if dataFormat == false
} SSmlHandle; } SSmlHandle;
//================================================================================================= //=================================================================================================
static volatile int64_t linesSmlHandleId = 0;
static const char* TS = "_ts";
static const char* TAG = "_tagNone";
//================================================================================================= //=================================================================================================
static volatile int64_t linesSmlHandleId = 0;
static int64_t smlGenId() { static int64_t smlGenId() {
int64_t id; int64_t id;
...@@ -154,6 +165,20 @@ static int64_t smlGenId() { ...@@ -154,6 +165,20 @@ static int64_t smlGenId() {
return id; return id;
} }
static inline bool smlDoubleToInt64OverFlow(double num) {
if(num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
return false;
}
static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashObj *pHash) {
void *val = taosHashGet(pHash, key, keyLen);
if (val) {
return true;
}
taosHashPut(pHash, key, keyLen, key, 1);
return false;
}
static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const char *msg2) { static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const char *msg2) {
if(msg1) strncat(pBuf->buf, msg1, pBuf->len); if(msg1) strncat(pBuf->buf, msg1, pBuf->len);
int32_t left = pBuf->len - strlen(pBuf->buf); int32_t left = pBuf->len - strlen(pBuf->buf);
...@@ -214,11 +239,11 @@ static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSi ...@@ -214,11 +239,11 @@ static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSi
memcpy(tname, field->key, field->keyLen); memcpy(tname, field->key, field->keyLen);
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
int32_t bytes = field->valueLen; // todo int32_t bytes = field->valueLen; // todo
int out = snprintf(buf, bufSize,"%s %s(%d)", int out = snprintf(buf, bufSize,"`%s` %s(%d)",
tname,tDataTypes[field->type].name, bytes); tname,tDataTypes[field->type].name, bytes);
*outBytes = out; *outBytes = out;
} else { } else {
int out = snprintf(buf, bufSize, "%s %s", tname, tDataTypes[type].name); int out = snprintf(buf, bufSize, "`%s` %s", tname, tDataTypes[type].name);
*outBytes = out; *outBytes = out;
} }
...@@ -327,7 +352,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { ...@@ -327,7 +352,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) {
break; break;
} }
case SCHEMA_ACTION_CREATE_STABLE: { case SCHEMA_ACTION_CREATE_STABLE: {
int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); int n = sprintf(result, "create stable `%s` (", action->createSTable.sTableName);
char* pos = result + n; int freeBytes = capacity - n; char* pos = result + n; int freeBytes = capacity - n;
SArray *cols = action->createSTable.fields; SArray *cols = action->createSTable.fields;
...@@ -480,294 +505,99 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { ...@@ -480,294 +505,99 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) {
// *pos = cur; // *pos = cur;
//} //}
static bool smlParseTinyInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){
const char *pVal = kvVal->value; const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen; int32_t len = kvVal->valueLen;
if (len <= 2) {
return false;
}
const char *signalPos = pVal + len - 2;
if (!strncasecmp(signalPos, "i8", 2)) {
char *endptr = NULL; char *endptr = NULL;
int64_t result = strtoll(pVal, &endptr, 10); double result = strtod(pVal, &endptr);
if(endptr != signalPos){ // 78ri8 if(pVal == endptr){
*isValid = false; smlBuildInvalidDataMsg(msg, "invalid data", pVal);
smlBuildInvalidDataMsg(msg, "invalid tiny int", endptr);
}else if(!IS_VALID_TINYINT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", endptr);
}else{
kvVal->i = result;
*isValid = true;
}
return true;
}
return false;
}
static bool smlParseTinyUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
if (len <= 2) {
return false;
}
if (pVal[0] == '-') {
return false; return false;
} }
const char *signalPos = pVal + len - 2;
if (!strncasecmp(signalPos, "u8", 2)) {
char *endptr = NULL;
int64_t result = strtoll(pVal, &endptr, 10);
if(endptr != signalPos){ // 78ri8
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid unsigned tiny int", endptr);
}else if(!IS_VALID_UTINYINT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", endptr);
}else{
kvVal->i = result;
*isValid = true;
}
return true;
}
return false;
}
static bool smlParseSmallInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { int32_t left = len - (endptr - pVal);
const char *pVal = kvVal->value; if(left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)){
int32_t len = kvVal->valueLen; kvVal->type = TSDB_DATA_TYPE_DOUBLE;
if (len <= 3) { kvVal->d = result;
}else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)){
if(!IS_VALID_FLOAT(result)){
smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal);
return false; return false;
} }
const char *signalPos = pVal + len - 3; kvVal->type = TSDB_DATA_TYPE_FLOAT;
if (!strncasecmp(signalPos, "i16", 3)) { kvVal->f = (float)result;
char *endptr = NULL; }else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)){
int64_t result = strtoll(pVal, &endptr, 10); if(smlDoubleToInt64OverFlow(result)){
if(endptr != signalPos){ // 78ri8 smlBuildInvalidDataMsg(msg, "big int is too large, out of precision", pVal);
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid small int", endptr);
}else if(!IS_VALID_SMALLINT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", endptr);
}else{
kvVal->i = result;
*isValid = true;
}
return true;
}
return false;
}
static bool smlParseSmallUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
if (len <= 3) {
return false; return false;
} }
if (pVal[0] == '-') { kvVal->type = TSDB_DATA_TYPE_BIGINT;
kvVal->i = (int64_t)result;
}else if ((left == 3 && strncasecmp(endptr, "u64", left) == 0)){
if(result >= (double)UINT64_MAX || result < 0){
smlBuildInvalidDataMsg(msg, "unsigned big int is too large, out of precision", pVal);
return false; return false;
} }
const char *signalPos = pVal + len - 3; kvVal->type = TSDB_DATA_TYPE_UBIGINT;
if (strncasecmp(signalPos, "u16", 3) == 0) { kvVal->u = result;
char *endptr = NULL; }else if (left == 3 && strncasecmp(endptr, "i32", left) == 0){
int64_t result = strtoll(pVal, &endptr, 10); if(!IS_VALID_INT(result)){
if(endptr != signalPos){ // 78ri8 smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal);
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid unsigned small int", endptr);
}else if(!IS_VALID_USMALLINT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", endptr);
}else{
kvVal->i = result;
*isValid = true;
}
return true;
}
return false;
}
static bool smlParseInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
if (len <= 3) {
return false; return false;
} }
const char *signalPos = pVal + len - 3; kvVal->type = TSDB_DATA_TYPE_INT;
if (strncasecmp(signalPos, "i32", 3) == 0) {
char *endptr = NULL;
int64_t result = strtoll(pVal, &endptr, 10);
if(endptr != signalPos){ // 78ri8
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid int", endptr);
}else if(!IS_VALID_INT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", endptr);
}else{
kvVal->i = result; kvVal->i = result;
*isValid = true; }else if (left == 3 && strncasecmp(endptr, "u32", left) == 0){
} if(!IS_VALID_UINT(result)){
return true; smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal);
}
return false;
}
static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
if (len <= 3) {
return false; return false;
} }
if (pVal[0] == '-') { kvVal->type = TSDB_DATA_TYPE_UINT;
kvVal->u = result;
}else if (left == 3 && strncasecmp(endptr, "i16", left) == 0){
if(!IS_VALID_SMALLINT(result)){
smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal);
return false; return false;
} }
const char *signalPos = pVal + len - 3; kvVal->type = TSDB_DATA_TYPE_SMALLINT;
if (strncasecmp(signalPos, "u32", 3) == 0) {
char *endptr = NULL;
int64_t result = strtoll(pVal, &endptr, 10);
if(endptr != signalPos){ // 78ri8
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid unsigned int", endptr);
}else if(!IS_VALID_UINT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", endptr);
}else{
kvVal->i = result; kvVal->i = result;
*isValid = true; }else if (left == 3 && strncasecmp(endptr, "u16", left) == 0){
} if(!IS_VALID_USMALLINT(result)){
return true; smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal);
}
return false; return false;
}
static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
if (len > 3 && strncasecmp(pVal + len - 3, "i64", 3) == 0) {
char *endptr = NULL;
errno = 0;
int64_t result = strtoll(pVal, &endptr, 10);
if(endptr != pVal + len - 3){ // 78ri8
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid big int", endptr);
}else if(errno == ERANGE || !IS_VALID_BIGINT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr);
}else{
kvVal->i = result;
*isValid = true;
}
return true;
}else if (len > 1 && pVal[len - 1] == 'i') {
char *endptr = NULL;
errno = 0;
int64_t result = strtoll(pVal, &endptr, 10);
if(endptr != pVal + len - 1){ // 78ri8
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid big int", endptr);
}else if(errno == ERANGE || !IS_VALID_BIGINT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr);
}else{
kvVal->i = result;
*isValid = true;
}
return true;
} }
return false; kvVal->type = TSDB_DATA_TYPE_USMALLINT;
} kvVal->u = result;
}else if (left == 2 && strncasecmp(endptr, "i8", left) == 0){
static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { if(!IS_VALID_TINYINT(result)){
const char *pVal = kvVal->value; smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal);
int32_t len = kvVal->valueLen;
if (len <= 3) {
return false; return false;
} }
if (pVal[0] == '-') { kvVal->type = TSDB_DATA_TYPE_TINYINT;
kvVal->i = result;
}else if (left == 2 && strncasecmp(endptr, "u8", left) == 0){
if(!IS_VALID_UTINYINT(result)){
smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal);
return false; return false;
} }
const char *signalPos = pVal + len - 3; kvVal->type = TSDB_DATA_TYPE_UTINYINT;
if (strncasecmp(signalPos, "u64", 3) == 0) {
char *endptr = NULL;
errno = 0;
uint64_t result = strtoull(pVal, &endptr, 10);
if(endptr != signalPos){ // 78ri8
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid unsigned big int", endptr);
}else if(errno == ERANGE || !IS_VALID_UBIGINT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr);
}else{
kvVal->u = result; kvVal->u = result;
*isValid = true;
}
return true;
}
return false;
}
static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
char *endptr = NULL;
errno = 0;
float result = strtof(pVal, &endptr);
if(endptr == pVal + len && errno != ERANGE && IS_VALID_FLOAT(result)){ // 78
kvVal->f = result;
*isValid = true;
return true;
}
if (len > 3 && strncasecmp(pVal + len - 3, "f32", 3) == 0) {
if(endptr != pVal + len - 3){ // 78ri8
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid float", endptr);
}else if(errno == ERANGE || !IS_VALID_FLOAT(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", endptr);
}else{ }else{
kvVal->f = result; smlBuildInvalidDataMsg(msg, "invalid data", pVal);
*isValid = true;
}
return true;
}
return false;
}
static bool smlParseDouble(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
if (len <= 3) {
return false; return false;
} }
const char *signalPos = pVal + len - 3;
if (strncasecmp(signalPos, "f64", 3) == 0) {
char *endptr = NULL;
errno = 0;
double result = strtod(pVal, &endptr);
if(endptr != signalPos){ // 78ri8
*isValid = false;
smlBuildInvalidDataMsg(msg, "invalid double", endptr);
}else if(errno == ERANGE || !IS_VALID_DOUBLE(result)){
*isValid = false;
smlBuildInvalidDataMsg(msg, "double out of range[-1.7976931348623158e+308,1.7976931348623158e+308]", endptr);
}else{
kvVal->d = result;
*isValid = true;
}
return true; return true;
}
return false;
} }
static bool smlParseBool(SSmlKv *kvVal) { static bool smlParseBool(SSmlKv *kvVal) {
const char *pVal = kvVal->value; const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen; int32_t len = kvVal->valueLen;
if ((len == 1) && pVal[len - 1] == 't') { if ((len == 1) && pVal[0] == 't') {
kvVal->i = true; kvVal->i = true;
return true; return true;
} }
if ((len == 1) && pVal[len - 1] == 'f') { if ((len == 1) && pVal[0] == 'f') {
kvVal->i = false; kvVal->i = false;
return true; return true;
} }
...@@ -805,102 +635,181 @@ static bool smlIsNchar(const char *pVal, uint16_t len) { ...@@ -805,102 +635,181 @@ static bool smlIsNchar(const char *pVal, uint16_t len) {
return false; return false;
} }
static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) {
// put high probability matching type first char *endPtr = NULL;
bool isValid = false; double ts = (double)strtoll(value, &endPtr, 10);
if(value + len != endPtr){
//binary return -1;
if (smlIsBinary(pVal->value, pVal->valueLen)) {
pVal->type = TSDB_DATA_TYPE_BINARY;
pVal->valueLen -= 2;
pVal->length = pVal->valueLen;
pVal->value++;
return true;
} }
//nchar switch (type) {
if (smlIsNchar(pVal->value, pVal->valueLen)) { case TSDB_TIME_PRECISION_HOURS:
pVal->type = TSDB_DATA_TYPE_NCHAR; ts *= (3600 * 1e9);
pVal->valueLen -= 3; break;
pVal->length = pVal->valueLen; case TSDB_TIME_PRECISION_MINUTES:
pVal->value += 2; ts *= (60 * 1e9);
return true; break;
case TSDB_TIME_PRECISION_SECONDS:
ts *= (1e9);
break;
case TSDB_TIME_PRECISION_MILLI:
ts *= (1e6);
break;
case TSDB_TIME_PRECISION_MICRO:
ts *= (1e3);
break;
case TSDB_TIME_PRECISION_NANO:
break;
default:
ASSERT(0);
} }
//float if(ts >= (double)INT64_MAX || ts <= 0){
if (smlParseFloat(pVal, &isValid, msg)) { return -1;
if(!isValid) return false;
pVal->type = TSDB_DATA_TYPE_FLOAT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
return true;
} }
//double
if (smlParseDouble(pVal, &isValid, msg)) {
if(!isValid) return false;
pVal->type = TSDB_DATA_TYPE_DOUBLE;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
return true; return (int64_t)ts;
} }
//bool
if (smlParseBool(pVal)) { static int64_t smlGetTimeNow(int8_t precision) {
pVal->type = TSDB_DATA_TYPE_BOOL; switch (precision) {
pVal->length = (int16_t)tDataTypes[pVal->type].bytes; case TSDB_TIME_PRECISION_HOURS:
return true; return taosGetTimestampMs()/1000/3600;
case TSDB_TIME_PRECISION_MINUTES:
return taosGetTimestampMs()/1000/60;
case TSDB_TIME_PRECISION_SECONDS:
return taosGetTimestampMs()/1000;
case TSDB_TIME_PRECISION_MILLI:
case TSDB_TIME_PRECISION_MICRO:
case TSDB_TIME_PRECISION_NANO:
return taosGetTimestamp(precision);
default:
ASSERT(0);
} }
}
if (smlParseTinyInt(pVal, &isValid, msg)) { static int8_t smlGetTsTypeByLen(int32_t len) {
if(!isValid) return false; if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
pVal->type = TSDB_DATA_TYPE_TINYINT; return TSDB_TIME_PRECISION_SECONDS;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes; } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
return true; return TSDB_TIME_PRECISION_MILLI_DIGITS;
} else {
return -1;
} }
if (smlParseTinyUint(pVal, &isValid, msg)) { }
if(!isValid) return false;
pVal->type = TSDB_DATA_TYPE_UTINYINT; static int8_t smlGetTsTypeByPrecision(int8_t precision) {
pVal->length = (int16_t)tDataTypes[pVal->type].bytes; switch (precision) {
return true; case TSDB_SML_TIMESTAMP_HOURS:
return TSDB_TIME_PRECISION_HOURS;
case TSDB_SML_TIMESTAMP_MILLI_SECONDS:
return TSDB_TIME_PRECISION_MILLI;
case TSDB_SML_TIMESTAMP_NANO_SECONDS:
case TSDB_SML_TIMESTAMP_NOT_CONFIGURED:
return TSDB_TIME_PRECISION_NANO;
case TSDB_SML_TIMESTAMP_MICRO_SECONDS:
return TSDB_TIME_PRECISION_MICRO;
case TSDB_SML_TIMESTAMP_SECONDS:
return TSDB_TIME_PRECISION_SECONDS;
case TSDB_SML_TIMESTAMP_MINUTES:
return TSDB_TIME_PRECISION_MINUTES;
default:
return -1;
} }
if (smlParseSmallInt(pVal, &isValid, msg)) { }
if(!isValid) return false;
pVal->type = TSDB_DATA_TYPE_SMALLINT; static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){
pVal->length = (int16_t)tDataTypes[pVal->type].bytes; int8_t tsType = smlGetTsTypeByPrecision(info->precision);
return true; if (tsType == -1) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL);
return -1;
} }
if (smlParseSmallUint(pVal, &isValid, msg)) { if(!data){
if(!isValid) return false; return smlGetTimeNow(tsType);
pVal->type = TSDB_DATA_TYPE_USMALLINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
return true;
} }
if (smlParseInt(pVal, &isValid, msg)) {
if(!isValid) return false; int64_t ts = smlGetTimeValue(data, len, tsType);
pVal->type = TSDB_DATA_TYPE_INT; if(ts == -1){
pVal->length = (int16_t)tDataTypes[pVal->type].bytes; smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data);
return true; return -1;
} }
if (smlParseUint(pVal, &isValid, msg)) { return ts;
if(!isValid) return false; }
pVal->type = TSDB_DATA_TYPE_UINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes; static int64_t smlParseOpenTsdbTime(SSmlHandle* info, const char* data, int32_t len){
if(!data){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL);
return -1;
}
int8_t tsType = smlGetTsTypeByLen(len);
if (tsType == -1) {
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data);
return -1;
}
int64_t ts = smlGetTimeValue(data, len, tsType);
if(ts == -1){
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data);
return -1;
}
return ts;
}
static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArray *cols){
int64_t ts = 0;
if(info->protocol == TSDB_SML_LINE_PROTOCOL){
ts = smlParseInfluxTime(info, data, len);
}else{
ts = smlParseOpenTsdbTime(info, data, len);
}
if(ts == -1) return TSDB_CODE_TSC_INVALID_TIME_STAMP;
// add ts to
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if(!kv){
return TSDB_CODE_OUT_OF_MEMORY;
}
kv->key = TS;
kv->keyLen = TS_LEN;
kv->i = ts;
kv->type = TSDB_DATA_TYPE_TIMESTAMP;
kv->length = (int16_t)tDataTypes[kv->type].bytes;
if(cols) taosArrayPush(cols, &kv);
return TSDB_CODE_SUCCESS;
}
static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
//binary
if (smlIsBinary(pVal->value, pVal->valueLen)) {
pVal->type = TSDB_DATA_TYPE_BINARY;
pVal->valueLen -= BINARY_ADD_LEN;
pVal->length = pVal->valueLen;
pVal->value += (BINARY_ADD_LEN - 1);
return true; return true;
} }
if (smlParseBigInt(pVal, &isValid, msg)) { //nchar
if(!isValid) return false; if (smlIsNchar(pVal->value, pVal->valueLen)) {
pVal->type = TSDB_DATA_TYPE_BIGINT; pVal->type = TSDB_DATA_TYPE_NCHAR;
pVal->valueLen -= NCHAR_ADD_LEN;
pVal->length = pVal->valueLen;
pVal->value += (NCHAR_ADD_LEN - 1);
return true;
}
//bool
if (smlParseBool(pVal)) {
pVal->type = TSDB_DATA_TYPE_BOOL;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes; pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
return true; return true;
} }
if (smlParseBigUint(pVal, &isValid, msg)) { //number
if(!isValid) return false; if (smlParseNumber(pVal, msg)) {
pVal->type = TSDB_DATA_TYPE_UBIGINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes; pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
return true; return true;
} }
smlBuildInvalidDataMsg(msg, "invalid data", pVal->value);
return false; return false;
} }
static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){ static int32_t smlParseInfluxString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){
if(!sql) return TSDB_CODE_SML_INVALID_DATA; if(!sql) return TSDB_CODE_SML_INVALID_DATA;
while (*sql != '\0') { // jump the space at the begining while (*sql != '\0') { // jump the space at the begining
if(*sql != SPACE) { if(*sql != SPACE) {
...@@ -989,13 +898,137 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu ...@@ -989,13 +898,137 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void smlParseTelnetElement(const char **sql, const char **data, int32_t *len){
while (**sql != '\0') {
if(**sql != SPACE && !(*data)) {
*data = *sql;
}else if (**sql == SPACE && *data) {
*len = *sql - *data;
break;
}
(*sql)++;
}
}
static int32_t smlParseTelnetTags(const char* data, int32_t len, SArray *cols, SHashObj *dumplicateKey, SSmlMsgBuf *msg){
for(int i = 0; i < len; i++){
// parse key
const char *key = data + i;
int32_t keyLen = 0;
while(i < len){
if(data[i] == EQUAL){
keyLen = data + i - key;
break;
}
i++;
}
if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){
smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key);
return TSDB_CODE_SML_INVALID_DATA;
}
if(smlCheckDuplicateKey(key, keyLen, dumplicateKey)){
smlBuildInvalidDataMsg(msg, "dumplicate key", key);
return TSDB_CODE_TSC_DUP_TAG_NAMES;
}
// parse value
i++;
const char *value = data + i;
while(i < len){
if(data[i] == SPACE){
break;
}
i++;
}
int32_t valueLen = data + i - value;
if(valueLen == 0){
smlBuildInvalidDataMsg(msg, "invalid value", value);
return TSDB_CODE_SML_INVALID_DATA;
}
// add kv to SSmlKv
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if(!kv) return TSDB_CODE_OUT_OF_MEMORY;
kv->key = key;
kv->keyLen = keyLen;
kv->value = value;
kv->valueLen = valueLen;
kv->type = TSDB_DATA_TYPE_NCHAR;
if(cols) taosArrayPush(cols, &kv);
}
return TSDB_CODE_SUCCESS;
}
// format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTableInfo *tinfo, SArray *cols){
if(!sql) return TSDB_CODE_SML_INVALID_DATA;
// parse metric
smlParseTelnetElement(&sql, &tinfo->sTableName, &tinfo->sTableNameLen);
if (!(tinfo->sTableName) || tinfo->sTableNameLen == 0) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
// parse timestamp
const char *timestamp = NULL;
int32_t tLen = 0;
smlParseTelnetElement(&sql, &timestamp, &tLen);
if (!timestamp || tLen == 0) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
int32_t ret = smlParseTS(info, timestamp, tLen, cols);
if (ret != TSDB_CODE_SUCCESS) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
// parse value
const char *value = NULL;
int32_t valueLen = 0;
smlParseTelnetElement(&sql, &value, &valueLen);
if (!value || valueLen == 0) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if(!kv) return TSDB_CODE_OUT_OF_MEMORY;
taosArrayPush(cols, &kv);
kv->key = VALUE;
kv->keyLen = VALUE_LEN;
kv->value = value;
kv->valueLen = valueLen;
if(!smlParseValue(kv, &info->msgBuf) || kv->type == TSDB_DATA_TYPE_BINARY
|| kv->type == TSDB_DATA_TYPE_NCHAR || kv->type == TSDB_DATA_TYPE_BOOL){
return TSDB_CODE_SML_INVALID_DATA;
}
// parse tags
while(*sql == SPACE){
sql++;
}
ret = smlParseTelnetTags(sql, strlen(sql), tinfo->tags, info->dumplicateKey, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){
if(isTag && len == 0){ if(isTag && len == 0){
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if(!kv) return TSDB_CODE_OUT_OF_MEMORY;
kv->key = TAG; kv->key = TAG;
kv->keyLen = strlen(TAG); kv->keyLen = TAG_LEN;
kv->value = TAG; kv->value = TAG;
kv->valueLen = strlen(TAG); kv->valueLen = TAG_LEN;
kv->type = TSDB_DATA_TYPE_NCHAR; kv->type = TSDB_DATA_TYPE_NCHAR;
if(cols) taosArrayPush(cols, &kv); if(cols) taosArrayPush(cols, &kv);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -1017,11 +1050,9 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is ...@@ -1017,11 +1050,9 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
if(taosHashGet(dumplicateKey, key, keyLen)){ if(smlCheckDuplicateKey(key, keyLen, dumplicateKey)){
smlBuildInvalidDataMsg(msg, "dumplicate key", key); smlBuildInvalidDataMsg(msg, "dumplicate key", key);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_TSC_DUP_TAG_NAMES;
}else{
taosHashPut(dumplicateKey, key, keyLen, key, CHAR_BYTES);
} }
// parse value // parse value
...@@ -1029,7 +1060,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is ...@@ -1029,7 +1060,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
const char *value = data + i; const char *value = data + i;
bool isInQuote = false; bool isInQuote = false;
while(i < len){ while(i < len){
if(data[i] == QUOTE && data[i-1] != SLASH){ if(!isTag && data[i] == QUOTE && data[i-1] != SLASH){
isInQuote = !isInQuote; isInQuote = !isInQuote;
} }
if(!isInQuote && data[i] == COMMA && i > 0 && data[i-1] != SLASH){ if(!isInQuote && data[i] == COMMA && i > 0 && data[i-1] != SLASH){
...@@ -1037,7 +1068,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is ...@@ -1037,7 +1068,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
} }
i++; i++;
} }
if(isInQuote){ if(!isTag && isInQuote){
smlBuildInvalidDataMsg(msg, "only one quote", value); smlBuildInvalidDataMsg(msg, "only one quote", value);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
...@@ -1049,6 +1080,9 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is ...@@ -1049,6 +1080,9 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
// add kv to SSmlKv // add kv to SSmlKv
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if(!kv) return TSDB_CODE_OUT_OF_MEMORY;
if(cols) taosArrayPush(cols, &kv);
kv->key = key; kv->key = key;
kv->keyLen = keyLen; kv->keyLen = keyLen;
kv->value = value; kv->value = value;
...@@ -1060,216 +1094,73 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is ...@@ -1060,216 +1094,73 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
} }
if(cols) taosArrayPush(cols, &kv);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { //static int32_t parseSmlCols(const char* data, SArray *cols){
char *endPtr = NULL; // while(*data != '\0'){
double ts = (double)strtoll(value, &endPtr, 10); // if(*data == EQUAL) return TSDB_CODE_SML_INVALID_DATA;
if(value + len != endPtr){ // const char *key = data;
return -1; // int32_t keyLen = 0;
// while(*data != '\0'){
// if(*data == EQUAL && *(data-1) != SLASH){
// keyLen = data - key;
// data ++;
// break;
// }
// data++;
// }
// if(keyLen == 0){
// return TSDB_CODE_SML_INVALID_DATA;
// }
//
// if(*data == COMMA) return TSDB_CODE_SML_INVALID_DATA;
// const char *value = data;
// int32_t valueLen = 0;
// while(*data != '\0'){
// if(*data == COMMA && *(data-1) != SLASH){
// valueLen = data - value;
// data ++;
// break;
// }
// data++;
// }
// if(valueLen == 0){
// return TSDB_CODE_SML_INVALID_DATA;
// }
//
// TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1);
// kv->key = key;
// kv->keyLen = keyLen;
// kv->value = value;
// kv->valueLen = valueLen;
// kv->type = TSDB_DATA_TYPE_NCHAR;
// if(cols) taosArrayPush(cols, &kv);
// }
// return TSDB_CODE_SUCCESS;
//}
static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SSmlMsgBuf *msg){
if(tags){
for (int i = 0; i < taosArrayGetSize(tags); ++i) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(tags, i);
ASSERT(kv->type == TSDB_DATA_TYPE_NCHAR);
uint8_t *index = (uint8_t *)taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen);
if(index){
SSmlKv **value = (SSmlKv **)taosArrayGet(tableMeta->tags, *index);
ASSERT((*value)->type == TSDB_DATA_TYPE_NCHAR);
if(kv->valueLen > (*value)->valueLen){ // tags type is nchar
*value = kv;
} }
switch (type) { }else{
case TSDB_TIME_PRECISION_HOURS: size_t tmp = taosArrayGetSize(tableMeta->tags);
ts *= (3600 * 1e9); ASSERT(tmp <= UINT8_MAX);
break; uint8_t size = tmp;
case TSDB_TIME_PRECISION_MINUTES: taosArrayPush(tableMeta->tags, &kv);
ts *= (60 * 1e9); taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &size, CHAR_BYTES);
break;
case TSDB_TIME_PRECISION_SECONDS:
ts *= (1e9);
break;
case TSDB_TIME_PRECISION_MILLI:
ts *= (1e6);
break;
case TSDB_TIME_PRECISION_MICRO:
ts *= (1e3);
break;
case TSDB_TIME_PRECISION_NANO:
break;
default:
ASSERT(0);
}
if(ts > (double)INT64_MAX || ts < 0){
return -1;
}
return (int64_t)ts;
}
static int64_t smlGetTimeNow(int8_t precision) {
switch (precision) {
case TSDB_TIME_PRECISION_HOURS:
return taosGetTimestampMs()/1000/3600;
case TSDB_TIME_PRECISION_MINUTES:
return taosGetTimestampMs()/1000/60;
case TSDB_TIME_PRECISION_SECONDS:
return taosGetTimestampMs()/1000;
case TSDB_TIME_PRECISION_MILLI:
case TSDB_TIME_PRECISION_MICRO:
case TSDB_TIME_PRECISION_NANO:
return taosGetTimestamp(precision);
default:
ASSERT(0);
}
}
static int8_t smlGetTsTypeByLen(int32_t len) {
if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
return TSDB_TIME_PRECISION_SECONDS;
} else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
return TSDB_TIME_PRECISION_MILLI_DIGITS;
} else {
return -1;
}
}
static int8_t smlGetTsTypeByPrecision(int8_t precision) {
switch (precision) {
case TSDB_SML_TIMESTAMP_HOURS:
return TSDB_TIME_PRECISION_HOURS;
case TSDB_SML_TIMESTAMP_MILLI_SECONDS:
return TSDB_TIME_PRECISION_MILLI;
case TSDB_SML_TIMESTAMP_NANO_SECONDS:
case TSDB_SML_TIMESTAMP_NOT_CONFIGURED:
return TSDB_TIME_PRECISION_NANO;
case TSDB_SML_TIMESTAMP_MICRO_SECONDS:
return TSDB_TIME_PRECISION_MICRO;
case TSDB_SML_TIMESTAMP_SECONDS:
return TSDB_TIME_PRECISION_SECONDS;
case TSDB_SML_TIMESTAMP_MINUTES:
return TSDB_TIME_PRECISION_MINUTES;
default:
return -1;
}
}
static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){
int8_t tsType = smlGetTsTypeByPrecision(info->precision);
if (tsType == -1) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL);
return -1;
}
if(!data){
return smlGetTimeNow(tsType);
}
int64_t ts = smlGetTimeValue(data, len, tsType);
if(ts == -1){
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data);
return -1;
}
return ts;
}
static int64_t smlParseOpenTsdbTime(SSmlHandle* info, const char* data, int32_t len){
if(!data){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL);
return -1;
}
int8_t tsType = smlGetTsTypeByLen(len);
if (tsType == -1) {
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data);
return -1;
}
int64_t ts = smlGetTimeValue(data, len, tsType);
if(ts == -1){
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data);
return -1;
}
return ts;
}
static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArray *cols){
int64_t ts = 0;
if(info->protocol == TSDB_SML_LINE_PROTOCOL){
ts = smlParseInfluxTime(info, data, len);
}else{
ts = smlParseOpenTsdbTime(info, data, len);
}
if(ts == -1) return TSDB_CODE_TSC_INVALID_TIME_STAMP;
// add ts to
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if(!kv){
return TSDB_CODE_OUT_OF_MEMORY;
}
kv->key = TS;
kv->keyLen = strlen(kv->key);
kv->i = ts;
kv->type = TSDB_DATA_TYPE_TIMESTAMP;
kv->length = (int16_t)tDataTypes[kv->type].bytes;
if(cols) taosArrayPush(cols, &kv);
return TSDB_CODE_SUCCESS;
}
//static int32_t parseSmlCols(const char* data, SArray *cols){
// while(*data != '\0'){
// if(*data == EQUAL) return TSDB_CODE_SML_INVALID_DATA;
// const char *key = data;
// int32_t keyLen = 0;
// while(*data != '\0'){
// if(*data == EQUAL && *(data-1) != SLASH){
// keyLen = data - key;
// data ++;
// break;
// }
// data++;
// }
// if(keyLen == 0){
// return TSDB_CODE_SML_INVALID_DATA;
// }
//
// if(*data == COMMA) return TSDB_CODE_SML_INVALID_DATA;
// const char *value = data;
// int32_t valueLen = 0;
// while(*data != '\0'){
// if(*data == COMMA && *(data-1) != SLASH){
// valueLen = data - value;
// data ++;
// break;
// }
// data++;
// }
// if(valueLen == 0){
// return TSDB_CODE_SML_INVALID_DATA;
// }
//
// TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1);
// kv->key = key;
// kv->keyLen = keyLen;
// kv->value = value;
// kv->valueLen = valueLen;
// kv->type = TSDB_DATA_TYPE_NCHAR;
// if(cols) taosArrayPush(cols, &kv);
// }
// return TSDB_CODE_SUCCESS;
//}
static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SSmlMsgBuf *msg){
if(tags){
for (int i = 0; i < taosArrayGetSize(tags); ++i) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(tags, i);
ASSERT(kv->type == TSDB_DATA_TYPE_NCHAR);
uint8_t *index = (uint8_t *)taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen);
if(index){
SSmlKv **value = (SSmlKv **)taosArrayGet(tableMeta->tags, *index);
ASSERT((*value)->type == TSDB_DATA_TYPE_NCHAR);
if(kv->valueLen > (*value)->valueLen){ // tags type is nchar
*value = kv;
}
}else{
size_t tmp = taosArrayGetSize(tableMeta->tags);
ASSERT(tmp <= UINT8_MAX);
uint8_t size = tmp;
taosArrayPush(tableMeta->tags, &kv);
taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &size, CHAR_BYTES);
} }
} }
} }
...@@ -1321,125 +1212,715 @@ static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols) ...@@ -1321,125 +1212,715 @@ static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols)
} }
} }
static SSmlTableInfo* smlBuildTableInfo(bool format){ static SSmlTableInfo* smlBuildTableInfo(){
SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1);
if(!tag){ if(!tag){
return NULL; return NULL;
} }
if(format){
tag->colsFormat = taosArrayInit(16, POINTER_BYTES);
if (tag->colsFormat == NULL) {
uError("SML:smlParseLine failed to allocate memory");
goto cleanup;
}
}else{
tag->cols = taosArrayInit(16, POINTER_BYTES); tag->cols = taosArrayInit(16, POINTER_BYTES);
if (tag->cols == NULL) { if (tag->cols == NULL) {
uError("SML:smlParseLine failed to allocate memory"); uError("SML:smlBuildTableInfo failed to allocate memory");
goto cleanup; goto cleanup;
} }
}
tag->tags = taosArrayInit(16, POINTER_BYTES); tag->tags = taosArrayInit(16, POINTER_BYTES);
if (tag->tags == NULL) { if (tag->tags == NULL) {
uError("SML:smlParseLine failed to allocate memory"); uError("SML:smlBuildTableInfo failed to allocate memory");
goto cleanup;
}
return tag;
cleanup:
taosMemoryFree(tag);
return NULL;
}
static void smlDestroyTableInfo(SSmlTableInfo *tag, bool format){
if(format){
for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){
SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i);
for (int j = 0; j < taosArrayGetSize(kvArray); ++j) {
void *p = taosArrayGetP(kvArray, j);
taosMemoryFree(p);
}
taosArrayDestroy(kvArray);
}
}else{
for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){
SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i);
void** p1 = (void**)taosHashIterate(kvHash, NULL);
while (p1) {
taosMemoryFree(*p1);
p1 = (void**)taosHashIterate(kvHash, p1);
}
taosHashCleanup(kvHash);
}
}
taosArrayDestroy(tag->cols);
taosArrayDestroy(tag->tags);
taosMemoryFree(tag);
}
static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){
if(dataFormat){
taosArrayPush(oneTable->cols, &cols);
return TSDB_CODE_SUCCESS;
}
SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if(!kvHash){
uError("SML:smlDealCols failed to allocate memory");
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
for(size_t i = 0; i < taosArrayGetSize(cols); i++){
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); // todo key need escape, like \=, because find by schema name later
}
taosArrayPush(oneTable->cols, &kvHash);
return TSDB_CODE_SUCCESS;
}
static SSmlSTableMeta* smlBuildSTableMeta(){
SSmlSTableMeta* meta = (SSmlSTableMeta*)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1);
if(!meta){
return NULL;
}
meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (meta->tagHash == NULL) {
uError("SML:smlBuildSTableMeta failed to allocate memory");
goto cleanup;
}
meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (meta->fieldHash == NULL) {
uError("SML:smlBuildSTableMeta failed to allocate memory");
goto cleanup;
}
meta->tags = taosArrayInit(32, POINTER_BYTES);
if (meta->tags == NULL) {
uError("SML:smlBuildSTableMeta failed to allocate memory");
goto cleanup;
}
meta->cols = taosArrayInit(32, POINTER_BYTES);
if (meta->cols == NULL) {
uError("SML:smlBuildSTableMeta failed to allocate memory");
goto cleanup;
}
return meta;
cleanup:
taosMemoryFree(meta);
return NULL;
}
static void smlDestroySTableMeta(SSmlSTableMeta *meta){
taosHashCleanup(meta->tagHash);
taosHashCleanup(meta->fieldHash);
taosArrayDestroy(meta->tags);
taosArrayDestroy(meta->cols);
taosMemoryFree(meta->tableMeta);
}
static void smlDestroyCols(SArray *cols) {
if (!cols) return;
for (int i = 0; i < taosArrayGetSize(cols); ++i) {
void *kv = taosArrayGet(cols, i);
taosMemoryFree(kv);
}
}
static void smlDestroyInfo(SSmlHandle* info){
if(!info) return;
qDestroyQuery(info->pQuery);
smlDestroyHandle(info->exec);
// destroy info->childTables
void** p1 = (void**)taosHashIterate(info->childTables, NULL);
while (p1) {
smlDestroyTableInfo((SSmlTableInfo*)(*p1), info->dataFormat);
p1 = (void**)taosHashIterate(info->childTables, p1);
}
taosHashCleanup(info->childTables);
// destroy info->superTables
p1 = (void**)taosHashIterate(info->superTables, NULL);
while (p1) {
smlDestroySTableMeta((SSmlSTableMeta*)(*p1));
p1 = (void**)taosHashIterate(info->superTables, p1);
}
taosHashCleanup(info->superTables);
// destroy info->pVgHash
taosHashCleanup(info->pVgHash);
taosHashCleanup(info->dumplicateKey);
taosMemoryFreeClear(info);
}
static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){
int32_t code = TSDB_CODE_SUCCESS;
SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle));
if (NULL == info) {
return NULL;
}
info->id = smlGenId();
info->pQuery = (SQuery *)taosMemoryCalloc(1, sizeof(SQuery));
if (NULL == info->pQuery) {
uError("SML:0x%"PRIx64" create info->pQuery error", info->id);
goto cleanup;
}
info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
info->pQuery->haveResultSet = false;
info->pQuery->msgType = TDMT_VND_SUBMIT;
info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT);
if(NULL == info->pQuery->pRoot){
uError("SML:0x%"PRIx64" create info->pQuery->pRoot error", info->id);
goto cleanup;
}
((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV;
info->taos = (STscObj *)taos;
code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
if(code != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" get catalog error %d", info->id, code);
goto cleanup;
}
info->precision = precision;
info->protocol = protocol;
info->dataFormat = dataFormat;
info->pRequest = request;
info->msgBuf.buf = info->pRequest->msgBuf;
info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE;
info->exec = smlInitHandle(info->pQuery);
info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if(!dataFormat){
info->colsContainer = taosArrayInit(32, POINTER_BYTES);
if(NULL == info->colsContainer){
uError("SML:0x%"PRIx64" create info failed", info->id);
goto cleanup;
}
}
if(NULL == info->exec || NULL == info->childTables
|| NULL == info->superTables || NULL == info->pVgHash
|| NULL == info->dumplicateKey){
uError("SML:0x%"PRIx64" create info failed", info->id);
goto cleanup; goto cleanup;
} }
return tag;
cleanup: return info;
taosMemoryFree(tag); cleanup:
return NULL; smlDestroyInfo(info);
return NULL;
}
/************* TSDB_SML_JSON_PROTOCOL function start **************/
static int32_t smlJsonCreateSring(const char **output, char *input, int32_t inputLen){
*output = (const char *)taosMemoryMalloc(inputLen);
if (*output == NULL){
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memcpy((void*)(*output), input, inputLen);
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo) {
cJSON *metric = cJSON_GetObjectItem(root, "metric");
if (!cJSON_IsString(metric)) {
return TSDB_CODE_TSC_INVALID_JSON;
}
tinfo->sTableNameLen = strlen(metric->valuestring);
if (tinfo->sTableNameLen >= TSDB_TABLE_NAME_LEN) {
uError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters in JSON", info->id, TSDB_TABLE_NAME_LEN - 1);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
return smlJsonCreateSring(&tinfo->sTableName, metric->valuestring, tinfo->sTableNameLen);
}
static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsVal) {
int32_t size = cJSON_GetArraySize(root);
if (size != OTD_JSON_SUB_FIELDS_NUM) {
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *value = cJSON_GetObjectItem(root, "value");
if (!cJSON_IsNumber(value)) {
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *type = cJSON_GetObjectItem(root, "type");
if (!cJSON_IsString(type)) {
return TSDB_CODE_TSC_INVALID_JSON;
}
double timeDouble = value->valuedouble;
if(smlDoubleToInt64OverFlow(timeDouble)){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
if(timeDouble <= 0){
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
size_t typeLen = strlen(type->valuestring);
if (typeLen == 1 && type->valuestring[0] == 's') {
//seconds
timeDouble = timeDouble * 1e9;
if(smlDoubleToInt64OverFlow(timeDouble)){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
*tsVal = timeDouble;
} else if (typeLen == 2 && type->valuestring[1] == 's') {
switch (type->valuestring[0]) {
case 'm':
//milliseconds
timeDouble = timeDouble * 1e6;
if(smlDoubleToInt64OverFlow(timeDouble)){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
*tsVal = timeDouble;
break;
case 'u':
//microseconds
timeDouble = timeDouble * 1e3;
if(smlDoubleToInt64OverFlow(timeDouble)){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
*tsVal = timeDouble;
break;
case 'n':
//nanoseconds
*tsVal = timeDouble;
break;
default:
return TSDB_CODE_TSC_INVALID_JSON;
}
} else {
return TSDB_CODE_TSC_INVALID_JSON;
}
return TSDB_CODE_SUCCESS;
}
static uint8_t smlGetTimestampLen(int64_t num) {
uint8_t len = 0;
while ((num /= 10) != 0) {
len++;
}
len++;
return len;
}
static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) {
//Timestamp must be the first KV to parse
int64_t tsVal = 0;
cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp");
if (cJSON_IsNumber(timestamp)) {
//timestamp value 0 indicates current system time
double timeDouble = timestamp->valuedouble;
if(smlDoubleToInt64OverFlow(timeDouble)){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
if(timeDouble <= 0){
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble);
if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) {
timeDouble = timeDouble * 1e9;
if(smlDoubleToInt64OverFlow(timeDouble)){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
tsVal = timeDouble;
} else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) {
timeDouble = timeDouble * 1e6;
if(smlDoubleToInt64OverFlow(timeDouble)){
smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
tsVal = timeDouble;
} else {
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
} else if (cJSON_IsObject(timestamp)) {
int32_t ret = smlParseTSFromJSONObj(info, timestamp, &tsVal);
if (ret != TSDB_CODE_SUCCESS) {
uError("SML:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id);
return ret;
}
} else {
return TSDB_CODE_TSC_INVALID_JSON;
}
// add ts to
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if(!kv){
return TSDB_CODE_OUT_OF_MEMORY;
}
kv->key = TS;
kv->keyLen = TS_LEN;
kv->i = tsVal;
kv->type = TSDB_DATA_TYPE_TIMESTAMP;
kv->length = (int16_t)tDataTypes[kv->type].bytes;
if(cols) taosArrayPush(cols, &kv);
return TSDB_CODE_SUCCESS;
}
static int32_t smlConvertJSONBool(SSmlKv *pVal, char* typeStr, cJSON *value) {
if (strcasecmp(typeStr, "bool") != 0) {
uError("OTD:invalid type(%s) for JSON Bool", typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
pVal->type = TSDB_DATA_TYPE_BOOL;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->i = value->valueint;
return TSDB_CODE_SUCCESS;
}
static int32_t smlConvertJSONNumber(SSmlKv *pVal, char* typeStr, cJSON *value) {
//tinyint
if (strcasecmp(typeStr, "i8") == 0 ||
strcasecmp(typeStr, "tinyint") == 0) {
if (!IS_VALID_TINYINT(value->valuedouble)) {
uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_TINYINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->i = value->valuedouble;
return TSDB_CODE_SUCCESS;
}
//smallint
if (strcasecmp(typeStr, "i16") == 0 ||
strcasecmp(typeStr, "smallint") == 0) {
if (!IS_VALID_SMALLINT(value->valuedouble)) {
uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_SMALLINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->i = value->valuedouble;
return TSDB_CODE_SUCCESS;
}
//int
if (strcasecmp(typeStr, "i32") == 0 ||
strcasecmp(typeStr, "int") == 0) {
if (!IS_VALID_INT(value->valuedouble)) {
uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_INT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->i = value->valuedouble;
return TSDB_CODE_SUCCESS;
}
//bigint
if (strcasecmp(typeStr, "i64") == 0 ||
strcasecmp(typeStr, "bigint") == 0) {
pVal->type = TSDB_DATA_TYPE_BIGINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
if(smlDoubleToInt64OverFlow(value->valuedouble)){
uError("OTD:JSON value(%f) cannot fit in type(big int)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->i = value->valuedouble;
return TSDB_CODE_SUCCESS;
}
//float
if (strcasecmp(typeStr, "f32") == 0 ||
strcasecmp(typeStr, "float") == 0) {
if (!IS_VALID_FLOAT(value->valuedouble)) {
uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_FLOAT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->f = value->valuedouble;
return TSDB_CODE_SUCCESS;
}
//double
if (strcasecmp(typeStr, "f64") == 0 ||
strcasecmp(typeStr, "double") == 0) {
pVal->type = TSDB_DATA_TYPE_DOUBLE;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->d = value->valuedouble;
return TSDB_CODE_SUCCESS;
}
//if reach here means type is unsupported
uError("OTD:invalid type(%s) for JSON Number", typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
static int32_t smlConvertJSONString(SSmlKv *pVal, char* typeStr, cJSON *value) {
if (strcasecmp(typeStr, "binary") == 0) {
pVal->type = TSDB_DATA_TYPE_BINARY;
} else if (strcasecmp(typeStr, "nchar") == 0) {
pVal->type = TSDB_DATA_TYPE_NCHAR;
} else {
uError("OTD:invalid type(%s) for JSON String", typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
pVal->length = (int16_t)strlen(value->valuestring);
pVal->valueLen = pVal->length;
return smlJsonCreateSring(&pVal->value, value->valuestring, pVal->valueLen);
}
static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) {
int32_t ret = TSDB_CODE_SUCCESS;
int32_t size = cJSON_GetArraySize(root);
if (size != OTD_JSON_SUB_FIELDS_NUM) {
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *value = cJSON_GetObjectItem(root, "value");
if (value == NULL) {
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *type = cJSON_GetObjectItem(root, "type");
if (!cJSON_IsString(type)) {
return TSDB_CODE_TSC_INVALID_JSON;
}
switch (value->type) {
case cJSON_True:
case cJSON_False: {
ret = smlConvertJSONBool(kv, type->valuestring, value);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
break;
}
case cJSON_Number: {
ret = smlConvertJSONNumber(kv, type->valuestring, value);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
break;
}
case cJSON_String: {
ret = smlConvertJSONString(kv, type->valuestring, value);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
break;
}
default:
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
switch (root->type) {
case cJSON_True:
case cJSON_False: {
kv->type = TSDB_DATA_TYPE_BOOL;
kv->length = (int16_t)tDataTypes[kv->type].bytes;
kv->i = root->valueint;
break;
}
case cJSON_Number: {
kv->type = TSDB_DATA_TYPE_DOUBLE;
kv->length = (int16_t)tDataTypes[kv->type].bytes;
kv->d = root->valuedouble;
break;
}
case cJSON_String: {
/* set default JSON type to binary/nchar according to
* user configured parameter tsDefaultJSONStrType
*/
char *tsDefaultJSONStrType = "binary"; //todo
smlConvertJSONString(kv, tsDefaultJSONStrType, root);
break;
}
case cJSON_Object: {
int32_t ret = smlParseValueFromJSONObj(root, kv);
if (ret != TSDB_CODE_SUCCESS) {
uError("OTD:Failed to parse value from JSON Obj");
return ret;
}
break;
}
default:
return TSDB_CODE_TSC_INVALID_JSON;
}
return TSDB_CODE_SUCCESS;
} }
static void smlDestroyBuildTableInfo(SSmlTableInfo *tag, bool format){ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) {
if(format){ cJSON *metricVal = cJSON_GetObjectItem(root, "value");
taosArrayDestroy(tag->colsFormat); if (metricVal == NULL) {
}else{ return TSDB_CODE_TSC_INVALID_JSON;
tag->cols = taosArrayInit(16, POINTER_BYTES);
for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){
SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i);
void** p1 = (void**)taosHashIterate(kvHash, NULL);
while (p1) {
taosMemoryFree(*p1);
p1 = (void**)taosHashIterate(kvHash, p1);
} }
taosHashCleanup(kvHash);
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if(!kv){
return TSDB_CODE_OUT_OF_MEMORY;
} }
if(cols) taosArrayPush(cols, &kv);
kv->key = VALUE;
kv->keyLen = VALUE_LEN;
int32_t ret = smlParseValueFromJSON(metricVal, kv);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
} }
taosArrayDestroy(tag->tags); return TSDB_CODE_SUCCESS;
taosMemoryFree(tag);
} }
static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, SHashObj *dumplicateKey, SSmlMsgBuf *msg) {
if(dataFormat){ int32_t ret = TSDB_CODE_SUCCESS;
taosArrayPush(oneTable->colsFormat, &cols);
return TSDB_CODE_SUCCESS; cJSON *tags = cJSON_GetObjectItem(root, "tags");
if (tags == NULL || tags->type != cJSON_Object) {
return TSDB_CODE_TSC_INVALID_JSON;
}
//handle child table name todo
// size_t childTableNameLen = strlen(tsSmlChildTableName);
// char childTbName[TSDB_TABLE_NAME_LEN] = {0};
// if (childTableNameLen != 0) {
// memcpy(childTbName, tsSmlChildTableName, childTableNameLen);
// cJSON *id = cJSON_GetObjectItem(tags, childTbName);
// if (id != NULL) {
// if (!cJSON_IsString(id)) {
// tscError("OTD:0x%"PRIx64" ID must be JSON string", info->id);
// return TSDB_CODE_TSC_INVALID_JSON;
// }
// size_t idLen = strlen(id->valuestring);
// *childTableName = tcalloc(idLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char));
// memcpy(*childTableName, id->valuestring, idLen);
// addEscapeCharToString(*childTableName, (int32_t)idLen);
//
// //check duplicate IDs
// cJSON_DeleteItemFromObject(tags, childTbName);
// id = cJSON_GetObjectItem(tags, childTbName);
// if (id != NULL) {
// return TSDB_CODE_TSC_DUP_TAG_NAMES;
// }
// }
// }
int32_t tagNum = cJSON_GetArraySize(tags);
for (int32_t i = 0; i < tagNum; ++i) {
cJSON *tag = cJSON_GetArrayItem(tags, i);
if (tag == NULL) {
return TSDB_CODE_TSC_INVALID_JSON;
}
//check duplicate keys
if (smlCheckDuplicateKey(tag->string, strlen(tag->string), dumplicateKey)) {
return TSDB_CODE_TSC_DUP_TAG_NAMES;
} }
SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); // add kv to SSmlKv
if(!kvHash){ SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
uError("SML:smlDealCols failed to allocate memory"); if(!kv) return TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_TSC_OUT_OF_MEMORY; if(pKVs) taosArrayPush(pKVs, &kv);
//key
kv->keyLen = strlen(tag->string);
if (kv->keyLen >= TSDB_COL_NAME_LEN) {
uError("OTD:Tag key cannot exceeds %d characters in JSON", TSDB_COL_NAME_LEN - 1);
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
ret = smlJsonCreateSring(&kv->key, tag->string, kv->keyLen);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
//value
ret = smlParseValueFromJSON(tag, kv);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
} }
for(size_t i = 0; i < taosArrayGetSize(cols); i++){
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); // todo key need escape, like \=, because find by schema name later
} }
taosArrayPush(oneTable->cols, &kvHash);
return TSDB_CODE_SUCCESS; return ret;
} }
static SSmlSTableMeta* smlBuildSTableMeta(){ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo, SArray *cols) {
SSmlSTableMeta* meta = (SSmlSTableMeta*)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); int32_t ret = TSDB_CODE_SUCCESS;
if(!meta){
return NULL; if (!cJSON_IsObject(root)) {
uError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id);
return TSDB_CODE_TSC_INVALID_JSON;
} }
meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (meta->tagHash == NULL) { int32_t size = cJSON_GetArraySize(root);
uError("SML:smlBuildSTableMeta failed to allocate memory"); //outmost json fields has to be exactly 4
goto cleanup; if (size != OTD_JSON_FIELDS_NUM) {
uError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size);
return TSDB_CODE_TSC_INVALID_JSON;
} }
meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); //Parse metric
if (meta->fieldHash == NULL) { ret = smlParseMetricFromJSON(info, root, tinfo);
uError("SML:smlBuildSTableMeta failed to allocate memory"); if (ret != TSDB_CODE_SUCCESS) {
goto cleanup; uError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id);
return ret;
} }
uDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id);
meta->tags = taosArrayInit(32, POINTER_BYTES); //Parse timestamp
if (meta->tags == NULL) { ret = smlParseTSFromJSON(info, root, cols);
uError("SML:smlBuildSTableMeta failed to allocate memory"); if (ret) {
goto cleanup; uError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id);
return ret;
} }
uDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id);
meta->cols = taosArrayInit(32, POINTER_BYTES); //Parse metric value
if (meta->cols == NULL) { ret = smlParseColsFromJSON(root, cols);
uError("SML:smlBuildSTableMeta failed to allocate memory"); if (ret) {
goto cleanup; uError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id);
return ret;
} }
return meta; uDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id);
cleanup: //Parse tags
taosMemoryFree(meta); ret = smlParseTagsFromJSON(root, tinfo->tags, info->dumplicateKey, &info->msgBuf);
return NULL; if (ret) {
} uError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id);
return ret;
}
uDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id);
static void smlDestroySTableMeta(SSmlSTableMeta *meta){ return TSDB_CODE_SUCCESS;
taosHashCleanup(meta->tagHash);
taosHashCleanup(meta->fieldHash);
taosArrayDestroy(meta->tags);
taosArrayDestroy(meta->cols);
taosMemoryFree(meta->tableMeta);
} }
/************* TSDB_SML_JSON_PROTOCOL function end **************/
static int32_t smlParseLine(SSmlHandle* info, const char* sql) {
static int32_t smlParseInfluxLine(SSmlHandle* info, const char* sql) {
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
int ret = smlParseString(sql, &elements, &info->msgBuf); int ret = smlParseInfluxString(sql, &elements, &info->msgBuf);
if(ret != TSDB_CODE_SUCCESS){ if(ret != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" smlParseString failed", info->id); uError("SML:0x%"PRIx64" smlParseInfluxLine failed", info->id);
return ret; return ret;
} }
...@@ -1447,7 +1928,7 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { ...@@ -1447,7 +1928,7 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) {
if(info->dataFormat){ // if dataFormat, cols need new memory to save data if(info->dataFormat){ // if dataFormat, cols need new memory to save data
cols = taosArrayInit(16, POINTER_BYTES); cols = taosArrayInit(16, POINTER_BYTES);
if (cols == NULL) { if (cols == NULL) {
uError("SML:0x%"PRIx64" smlParseLine failed to allocate memory", info->id); uError("SML:0x%"PRIx64" smlParseInfluxLine failed to allocate memory", info->id);
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
}else{ // if dataFormat is false, cols do not need to save data, there is another new memory to save data }else{ // if dataFormat is false, cols do not need to save data, there is another new memory to save data
...@@ -1457,11 +1938,14 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { ...@@ -1457,11 +1938,14 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) {
ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols);
if(ret != TSDB_CODE_SUCCESS){ if(ret != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" smlParseTS failed", info->id); uError("SML:0x%"PRIx64" smlParseTS failed", info->id);
if(info->dataFormat) taosArrayDestroy(cols);
return ret; return ret;
} }
ret = smlParseCols(elements.cols, elements.colsLen, cols, false, info->dumplicateKey, &info->msgBuf); ret = smlParseCols(elements.cols, elements.colsLen, cols, false, info->dumplicateKey, &info->msgBuf);
if(ret != TSDB_CODE_SUCCESS){ if(ret != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" smlParseCols parse cloums fields failed", info->id); uError("SML:0x%"PRIx64" smlParseCols parse cloums fields failed", info->id);
smlDestroyCols(cols);
if(info->dataFormat) taosArrayDestroy(cols);
return ret; return ret;
} }
if(taosArrayGetSize(cols) > TSDB_MAX_COLUMNS){ if(taosArrayGetSize(cols) > TSDB_MAX_COLUMNS){
...@@ -1469,63 +1953,58 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { ...@@ -1469,63 +1953,58 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) {
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
bool hasTable = true;
SSmlTableInfo *tinfo = NULL;
SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen);
if(oneTable){ if(!oneTable){
SSmlSTableMeta** tableMeta = (SSmlSTableMeta**)taosHashGet(info->superTables, elements.measure, elements.measureLen); tinfo = smlBuildTableInfo();
ASSERT(tableMeta);
ret = smlUpdateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta cols
if(!ret){
uError("SML:0x%"PRIx64" smlUpdateMeta cols failed", info->id);
return TSDB_CODE_SML_INVALID_DATA;
}
ret = smlDealCols(*oneTable, info->dataFormat, cols);
if(ret != TSDB_CODE_SUCCESS){
return ret;
}
}else{
SSmlTableInfo *tinfo = smlBuildTableInfo(info->dataFormat);
if(!tinfo){ if(!tinfo){
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
ret = smlDealCols(tinfo, info->dataFormat, cols); taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES);
oneTable = &tinfo;
hasTable = false;
}
ret = smlDealCols(*oneTable, info->dataFormat, cols);
if(ret != TSDB_CODE_SUCCESS){ if(ret != TSDB_CODE_SUCCESS){
return ret; return ret;
} }
ret = smlParseCols(elements.tags, elements.tagsLen, tinfo->tags, true, info->dumplicateKey, &info->msgBuf); if(!hasTable){
ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, true, info->dumplicateKey, &info->msgBuf);
if(ret != TSDB_CODE_SUCCESS){ if(ret != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" smlParseCols parse tag fields failed", info->id); uError("SML:0x%"PRIx64" smlParseCols parse tag fields failed", info->id);
return ret; return ret;
} }
if(taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS){ if(taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_TAGS){
smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
tinfo->sTableName = elements.measure; (*oneTable)->sTableName = elements.measure;
tinfo->sTableNameLen = elements.measureLen; (*oneTable)->sTableNameLen = elements.measureLen;
RandTableName rName = { tinfo->tags, tinfo->sTableName, tinfo->sTableNameLen, RandTableName rName = {.tags=(*oneTable)->tags, .sTableName=(*oneTable)->sTableName, .sTableNameLen=(uint8_t)(*oneTable)->sTableNameLen,
tinfo->childTableName, 0 }; .childTableName=(*oneTable)->childTableName};
buildChildTableName(&rName); buildChildTableName(&rName);
tinfo->uid = rName.uid; (*oneTable)->uid = rName.uid;
}
SSmlSTableMeta** tableMeta = (SSmlSTableMeta**)taosHashGet(info->superTables, elements.measure, elements.measureLen); SSmlSTableMeta** tableMeta = (SSmlSTableMeta**)taosHashGet(info->superTables, elements.measure, elements.measureLen);
if(tableMeta){ // update meta if(tableMeta){ // update meta
ret = smlUpdateMeta(*tableMeta, tinfo->tags, cols, &info->msgBuf); ret = smlUpdateMeta(*tableMeta, hasTable ? NULL : (*oneTable)->tags, cols, &info->msgBuf);
if(!ret){ if(!ret){
uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id); uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
}else{ }else{
SSmlSTableMeta *meta = smlBuildSTableMeta(); SSmlSTableMeta *meta = smlBuildSTableMeta();
smlInsertMeta(meta, tinfo->tags, cols); smlInsertMeta(meta, (*oneTable)->tags, cols);
taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES);
} }
taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES);
}
if(!info->dataFormat){ if(!info->dataFormat){
taosArrayClear(info->colsContainer); taosArrayClear(info->colsContainer);
} }
...@@ -1533,96 +2012,111 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { ...@@ -1533,96 +2012,111 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void smlDestroyInfo(SSmlHandle* info){ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) {
if(!info) return; int ret = TSDB_CODE_SUCCESS;
qDestroyQuery(info->pQuery); SSmlTableInfo *tinfo = smlBuildTableInfo();
smlDestroyHandle(info->exec); if(!tinfo){
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
// destroy info->childTables SArray *cols = taosArrayInit(16, POINTER_BYTES);
void** p1 = (void**)taosHashIterate(info->childTables, NULL); if (cols == NULL) {
while (p1) { uError("SML:0x%"PRIx64" smlParseTelnetLine failed to allocate memory", info->id);
smlDestroyBuildTableInfo((SSmlTableInfo*)(*p1), info->dataFormat); return TSDB_CODE_TSC_OUT_OF_MEMORY;
p1 = (void**)taosHashIterate(info->childTables, p1);
} }
taosHashCleanup(info->childTables);
// destroy info->superTables if(info->protocol == TSDB_SML_TELNET_PROTOCOL){
p1 = (void**)taosHashIterate(info->superTables, NULL); smlParseTelnetString(info, (const char*)data, tinfo, cols);
while (p1) { }else if(info->protocol == TSDB_SML_JSON_PROTOCOL){
smlDestroySTableMeta((SSmlSTableMeta*)(*p1)); smlParseJSONString(info, (cJSON *)data, tinfo, cols);
p1 = (void**)taosHashIterate(info->superTables, p1); }else{
ASSERT(0);
}
if(ret != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" smlParseTelnetLine failed", info->id);
smlDestroyTableInfo(tinfo, true);
taosArrayDestroy(cols);
return ret;
} }
taosHashCleanup(info->superTables);
// destroy info->pVgHash if(taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS){
taosHashCleanup(info->pVgHash); smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL);
taosHashCleanup(info->dumplicateKey); return TSDB_CODE_SML_INVALID_DATA;
}
taosHashClear(info->dumplicateKey);
taosMemoryFreeClear(info); RandTableName rName = {.tags=tinfo->tags, .sTableName=tinfo->sTableName, .sTableNameLen=(uint8_t)tinfo->sTableNameLen,
} .childTableName=tinfo->childTableName};
buildChildTableName(&rName);
tinfo->uid = rName.uid;
static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ bool hasTable = true;
int32_t code = TSDB_CODE_SUCCESS; SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName));
SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle)); if(!oneTable) {
if (NULL == info) { taosHashPut(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName), &tinfo, POINTER_BYTES);
return NULL; oneTable = &tinfo;
hasTable = false;
}else{
smlDestroyTableInfo(tinfo, true);
} }
info->id = smlGenId();
info->pQuery = (SQuery *)taosMemoryCalloc(1, sizeof(SQuery)); taosArrayPush((*oneTable)->cols, &cols);
if (NULL == info->pQuery) { SSmlSTableMeta** tableMeta = (SSmlSTableMeta** )taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen);
uError("SML:0x%"PRIx64" create info->pQuery error", info->id); if(tableMeta){ // update meta
goto cleanup; ret = smlUpdateMeta(*tableMeta, hasTable ? NULL : (*oneTable)->tags, cols, &info->msgBuf);
if(!ret){
uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id);
return TSDB_CODE_SML_INVALID_DATA;
} }
info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; }else{
info->pQuery->haveResultSet = false; SSmlSTableMeta *meta = smlBuildSTableMeta();
info->pQuery->msgType = TDMT_VND_SUBMIT; smlInsertMeta(meta, (*oneTable)->tags, cols);
info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES);
if(NULL == info->pQuery->pRoot){
uError("SML:0x%"PRIx64" create info->pQuery->pRoot error", info->id);
goto cleanup;
} }
((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV;
info->taos = (STscObj *)taos; return TSDB_CODE_SUCCESS;
code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); }
if(code != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" get catalog error %d", info->id, code);
goto cleanup;
}
info->precision = precision; static int32_t smlParseJSON(SSmlHandle *info, char* payload) {
info->protocol = protocol; int32_t payloadNum = 0;
info->dataFormat = dataFormat; int32_t ret = TSDB_CODE_SUCCESS;
info->pRequest = request;
info->msgBuf.buf = info->pRequest->msgBuf;
info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE;
info->exec = smlInitHandle(info->pQuery); if (payload == NULL) {
info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); uError("SML:0x%"PRIx64" empty JSON Payload", info->id);
info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); return TSDB_CODE_TSC_INVALID_JSON;
info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); }
info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); cJSON *root = cJSON_Parse(payload);
if(!dataFormat){ if (root == NULL) {
info->colsContainer = taosArrayInit(32, POINTER_BYTES); uError("SML:0x%"PRIx64" parse json failed:%s", info->id, payload);
if(NULL == info->colsContainer){ return TSDB_CODE_TSC_INVALID_JSON;
uError("SML:0x%"PRIx64" create info failed", info->id);
goto cleanup;
} }
//multiple data points must be sent in JSON array
if (cJSON_IsObject(root)) {
payloadNum = 1;
} else if (cJSON_IsArray(root)) {
payloadNum = cJSON_GetArraySize(root);
} else {
uError("SML:0x%"PRIx64" Invalid JSON Payload", info->id);
ret = TSDB_CODE_TSC_INVALID_JSON;
goto end;
}
for (int32_t i = 0; i < payloadNum; ++i) {
cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(root)) ? root : cJSON_GetArrayItem(root, i);
ret = smlParseTelnetLine(info, dataPoint);
if(ret != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" Invalid JSON Payload", info->id);
goto end;
} }
if(NULL == info->exec || NULL == info->childTables
|| NULL == info->superTables || NULL == info->pVgHash
|| NULL == info->dumplicateKey){
uError("SML:0x%"PRIx64" create info failed", info->id);
goto cleanup;
} }
return info; end:
cleanup: cJSON_Delete(root);
smlDestroyInfo(info); return ret;
return NULL;
} }
static int32_t smlInsertData(SSmlHandle* info) { static int32_t smlInsertData(SSmlHandle* info) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
...@@ -1636,7 +2130,7 @@ static int32_t smlInsertData(SSmlHandle* info) { ...@@ -1636,7 +2130,7 @@ static int32_t smlInsertData(SSmlHandle* info) {
SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
SVgroupInfo vg; SVgroupInfo vg;
code = catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg); code = catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg);
if (code != 0) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%"PRIx64" catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName); uError("SML:0x%"PRIx64" catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName);
return code; return code;
} }
...@@ -1649,15 +2143,19 @@ static int32_t smlInsertData(SSmlHandle* info) { ...@@ -1649,15 +2143,19 @@ static int32_t smlInsertData(SSmlHandle* info) {
(*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->vgId = vg.vgId;
(*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid
code = smlBindData(info->exec, tableData->tags, tableData->colsFormat, (*pMeta)->cols, code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat,
tableData->cols, info->dataFormat, (*pMeta)->tableMeta, tableData->childTableName, info->msgBuf.buf, info->msgBuf.len); (*pMeta)->tableMeta, tableData->childTableName, info->msgBuf.buf, info->msgBuf.len);
if(code != TSDB_CODE_SUCCESS){ if(code != TSDB_CODE_SUCCESS){
return code; return code;
} }
oneTable = (SSmlTableInfo**)taosHashIterate(info->childTables, oneTable); oneTable = (SSmlTableInfo**)taosHashIterate(info->childTables, oneTable);
} }
smlBuildOutput(info->exec, info->pVgHash); code = smlBuildOutput(info->exec, info->pVgHash);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%"PRIx64" smlBuildOutput failed", info->id);
return code;
}
info->cost.insertRpcTime = taosGetTimestampUs(); info->cost.insertRpcTime = taosGetTimestampUs();
launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true, NULL); launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true, NULL);
...@@ -1675,22 +2173,40 @@ static void smlPrintStatisticInfo(SSmlHandle *info){ ...@@ -1675,22 +2173,40 @@ static void smlPrintStatisticInfo(SSmlHandle *info){
info->cost.endTime-info->cost.parseTime); info->cost.endTime-info->cost.parseTime);
} }
static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { static int32_t smlParseLine(SSmlHandle *info, char* lines[], int numLines){
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (info->protocol == TSDB_SML_JSON_PROTOCOL) {
if (numLines <= 0 || numLines > 65536) { code = smlParseJSON(info, *lines);
uError("SML:0x%"PRIx64" smlInsertLines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); if (code != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_APP_ERROR; uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, *lines);
goto cleanup; return code;
}
} }
info->cost.parseTime = taosGetTimestampUs();
for (int32_t i = 0; i < numLines; ++i) { for (int32_t i = 0; i < numLines; ++i) {
code = smlParseLine(info, lines[i]); if(info->protocol == TSDB_SML_LINE_PROTOCOL){
code = smlParseInfluxLine(info, lines[i]);
}else if(info->protocol == TSDB_SML_TELNET_PROTOCOL){
code = smlParseTelnetLine(info, lines[i]);
}else{
ASSERT(0);
}
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%"PRIx64" smlParseLine failed. line %d : %s", info->id, i, lines[i]); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, lines[i]);
goto cleanup; return code;
}
} }
return code;
}
static int smlProcess(SSmlHandle *info, char* lines[], int numLines) {
int32_t code = TSDB_CODE_SUCCESS;
info->cost.parseTime = taosGetTimestampUs();
code = smlParseLine(info, lines, numLines);
if (code != 0) {
uError("SML:0x%"PRIx64" smlParseLine error : %s", info->id, tstrerror(code));
goto cleanup;
} }
info->cost.lineNum = numLines; info->cost.lineNum = numLines;
...@@ -1742,6 +2258,7 @@ cleanup: ...@@ -1742,6 +2258,7 @@ cleanup:
TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT);
if(!request){ if(!request){
uError("SML:taos_schemaless_insert error request is null");
return NULL; return NULL;
} }
...@@ -1750,22 +2267,28 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr ...@@ -1750,22 +2267,28 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
return (TAOS_RES*)request; return (TAOS_RES*)request;
} }
switch (protocol) { if (numLines <= 0 || numLines > 65536) {
case TSDB_SML_LINE_PROTOCOL:{ request->code = TSDB_CODE_SML_INVALID_DATA;
smlInsertLines(info, lines, numLines); smlBuildInvalidDataMsg(&info->msgBuf, "numLines should be between 1 and 65536", NULL);
break; goto end;
} }
case TSDB_SML_TELNET_PROTOCOL:
//code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); if(protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL){
break; request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
case TSDB_SML_JSON_PROTOCOL: smlBuildInvalidDataMsg(&info->msgBuf, "protocol invalidate", NULL);
//code = taos_insert_json_payload(taos, *lines, protocol, tsType, &affected_rows); goto end;
break;
default:
break;
} }
smlDestroyInfo(info);
if(protocol == TSDB_SML_LINE_PROTOCOL && (precision < TSDB_SML_TIMESTAMP_HOURS || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)){
request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE;
smlBuildInvalidDataMsg(&info->msgBuf, "precision invalidate for line protocol", NULL);
goto end;
}
info->pRequest->code = smlProcess(info, lines, numLines);
end:
smlDestroyInfo(info);
return (TAOS_RES*)request; return (TAOS_RES*)request;
} }
source/client/src/clientStmt.c
浏览文件 @ eb016e01
...@@ -279,6 +279,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { ...@@ -279,6 +279,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) {
} }
pStmt->exec.autoCreateTbl = false; pStmt->exec.autoCreateTbl = false;
pStmt->exec.emptyRes = false;
if (keepTable) { if (keepTable) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -628,8 +629,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { ...@@ -628,8 +629,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); STMT_ERR_RET(stmtRestoreQueryFields(pStmt));
} }
bool emptyResult = false; STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId, &pStmt->exec.emptyRes));
STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId, &emptyResult));
} }
STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
...@@ -736,7 +736,11 @@ int stmtExec(TAOS_STMT *stmt) { ...@@ -736,7 +736,11 @@ int stmtExec(TAOS_STMT *stmt) {
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
if (STMT_TYPE_QUERY == pStmt->sql.type) { if (STMT_TYPE_QUERY == pStmt->sql.type) {
if (pStmt->exec.emptyRes) {
pStmt->exec.pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
} else {
scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList, NULL); scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList, NULL);
}
} else { } else {
STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, (autoCreateTbl ? (void**)&pRsp : NULL)); launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, (autoCreateTbl ? (void**)&pRsp : NULL));
......
source/client/test/smlTest.cpp
浏览文件 @ eb016e01
...@@ -33,7 +33,7 @@ int main(int argc, char **argv) { ...@@ -33,7 +33,7 @@ int main(int argc, char **argv) {
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
} }
TEST(testCase, smlParseString_Test) { TEST(testCase, smlParseInfluxString_Test) {
char msg[256] = {0}; char msg[256] = {0};
SSmlMsgBuf msgBuf; SSmlMsgBuf msgBuf;
msgBuf.buf = msg; msgBuf.buf = msg;
...@@ -42,7 +42,7 @@ TEST(testCase, smlParseString_Test) { ...@@ -42,7 +42,7 @@ TEST(testCase, smlParseString_Test) {
// case 1 // case 1
char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3";
int ret = smlParseString(sql, &elements, &msgBuf); int ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
ASSERT_EQ(elements.measureLen, strlen("st")); ASSERT_EQ(elements.measureLen, strlen("st"));
...@@ -60,13 +60,13 @@ TEST(testCase, smlParseString_Test) { ...@@ -60,13 +60,13 @@ TEST(testCase, smlParseString_Test) {
// case 2 false // case 2 false
sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
// case 3 false // case 3 false
sql = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; sql = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2);
ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3"));
...@@ -74,7 +74,7 @@ TEST(testCase, smlParseString_Test) { ...@@ -74,7 +74,7 @@ TEST(testCase, smlParseString_Test) {
// case 4 tag is null // case 4 tag is null
sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000";
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
ASSERT_EQ(elements.measureLen, strlen("st")); ASSERT_EQ(elements.measureLen, strlen("st"));
...@@ -92,7 +92,7 @@ TEST(testCase, smlParseString_Test) { ...@@ -92,7 +92,7 @@ TEST(testCase, smlParseString_Test) {
// case 5 tag is null // case 5 tag is null
sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ";
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
sql++; sql++;
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
...@@ -111,13 +111,13 @@ TEST(testCase, smlParseString_Test) { ...@@ -111,13 +111,13 @@ TEST(testCase, smlParseString_Test) {
// case 6 // case 6
sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 ";
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
// case 7 // case 7
sql = " st , "; sql = " st , ";
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
sql++; sql++;
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3);
...@@ -126,7 +126,7 @@ TEST(testCase, smlParseString_Test) { ...@@ -126,7 +126,7 @@ TEST(testCase, smlParseString_Test) {
// case 8 false // case 8 false
sql = ", st , "; sql = ", st , ";
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
} }
...@@ -140,15 +140,13 @@ TEST(testCase, smlParseCols_Error_Test) { ...@@ -140,15 +140,13 @@ TEST(testCase, smlParseCols_Error_Test) {
"c=f64", // double "c=f64", // double
"c=8f64f", "c=8f64f",
"c=8ef64", "c=8ef64",
"c=1.7976931348623158e+390f64",
"c=f32", // float "c=f32", // float
"c=8f32f", "c=8f32f",
"c=8wef32", "c=8wef32",
"c=-3.402823466e+39f32", "c=-3.402823466e+39f32",
"c=", // float "c=", // double
"c=8f", "c=8f",
"c=8we", "c=8we",
"c=3.402823466e+39",
"c=i8", // tiny int "c=i8", // tiny int
"c=-8i8f", "c=-8i8f",
"c=8wei8", "c=8wei8",
...@@ -218,7 +216,7 @@ TEST(testCase, smlParseCols_tag_Test) { ...@@ -218,7 +216,7 @@ TEST(testCase, smlParseCols_tag_Test) {
SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
const char *data = const char *data =
"cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; "cbin=\"passit helloc=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\"";
int32_t len = strlen(data); int32_t len = strlen(data);
int32_t ret = smlParseCols(data, len, cols, true, dumplicateKey, &msgBuf); int32_t ret = smlParseCols(data, len, cols, true, dumplicateKey, &msgBuf);
ASSERT_EQ(ret, TSDB_CODE_SUCCESS); ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
...@@ -230,7 +228,7 @@ TEST(testCase, smlParseCols_tag_Test) { ...@@ -230,7 +228,7 @@ TEST(testCase, smlParseCols_tag_Test) {
ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->valueLen, 18); ASSERT_EQ(kv->valueLen, 17);
ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0);
taosMemoryFree(kv); taosMemoryFree(kv);
...@@ -280,7 +278,7 @@ TEST(testCase, smlParseCols_Test) { ...@@ -280,7 +278,7 @@ TEST(testCase, smlParseCols_Test) {
SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\"";
int32_t len = strlen(data); int32_t len = strlen(data);
int32_t ret = smlParseCols(data, len, cols, false, dumplicateKey, &msgBuf); int32_t ret = smlParseCols(data, len, cols, false, dumplicateKey, &msgBuf);
ASSERT_EQ(ret, TSDB_CODE_SUCCESS); ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
...@@ -321,17 +319,17 @@ TEST(testCase, smlParseCols_Test) { ...@@ -321,17 +319,17 @@ TEST(testCase, smlParseCols_Test) {
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->length, 8);
//ASSERT_EQ(kv->d, 4.31); //ASSERT_EQ(kv->d, 4.31);
printf("4.31 = kv->f:%f\n", kv->d); printf("4.31 = kv->d:%f\n", kv->d);
taosMemoryFree(kv); taosMemoryFree(kv);
// float // float
kv = (SSmlKv *)taosArrayGetP(cols, 4); kv = (SSmlKv *)taosArrayGetP(cols, 4);
ASSERT_EQ(strncasecmp(kv->key, "cf32_", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->length, 8);
//ASSERT_EQ(kv->f, 8.32); //ASSERT_EQ(kv->f, 8.32);
printf("8.32 = kv->f:%f\n", kv->f); printf("8.32 = kv->d:%f\n", kv->d);
taosMemoryFree(kv); taosMemoryFree(kv);
// float // float
...@@ -467,7 +465,7 @@ TEST(testCase, smlParseCols_Test) { ...@@ -467,7 +465,7 @@ TEST(testCase, smlParseCols_Test) {
taosHashCleanup(dumplicateKey); taosHashCleanup(dumplicateKey);
} }
TEST(testCase, smlParseLine_Test) { TEST(testCase, smlProcess_influx_Test) {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr); ASSERT_NE(taos, nullptr);
...@@ -483,7 +481,7 @@ TEST(testCase, smlParseLine_Test) { ...@@ -483,7 +481,7 @@ TEST(testCase, smlParseLine_Test) {
SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true);
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
const char *sql[9] = { const char *sql[11] = {
"readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0 1451606400000000000", "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0 1451606400000000000",
"readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607400000000000", "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607400000000000",
"readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,heading=221,grade=0,fuel_consumption=25 1451608400000000000", "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,heading=221,grade=0,fuel_consumption=25 1451608400000000000",
...@@ -492,14 +490,24 @@ TEST(testCase, smlParseLine_Test) { ...@@ -492,14 +490,24 @@ TEST(testCase, smlParseLine_Test) {
"readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000", "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000",
"readings,name=truck_2,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000", "readings,name=truck_2,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000",
"readings,name=truck_2,fleet=North,driver=Derek,model=F-150 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609400000000000", "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609400000000000",
"readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451629400000000000" "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451629400000000000",
"stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=3,c4=4 1451629500000000000",
"stable,t2=t2,t1=t1,t3=t3 c1=1,c3=3,c4=4 1451629600000000000"
}; };
smlInsertLines(info, (char**)sql, 9); smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
// for (int i = 0; i < 3; i++) {
// smlParseLine(info, sql[i]); TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d");
// } ASSERT_NE(res, nullptr);
int fieldNum = taos_field_count(res);
ASSERT_EQ(fieldNum, 11);
int rowNum = taos_affected_rows(res);
ASSERT_EQ(rowNum, 2);
for (int i = 0; i < rowNum; ++i) {
TAOS_ROW rows = taos_fetch_row(res);
}
} }
// different types
TEST(testCase, smlParseLine_error_Test) { TEST(testCase, smlParseLine_error_Test) {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr); ASSERT_NE(taos, nullptr);
...@@ -520,24 +528,247 @@ TEST(testCase, smlParseLine_error_Test) { ...@@ -520,24 +528,247 @@ TEST(testCase, smlParseLine_error_Test) {
"measure,t1=3 c1=8", "measure,t1=3 c1=8",
"measure,t2=3 c1=8u8" "measure,t2=3 c1=8u8"
}; };
int ret = smlInsertLines(info, (char **)sql, 2); int ret = smlProcess(info, (char **)sql, sizeof(sql)/sizeof(sql[0]));
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
} }
// TEST(testCase, smlParseTS_Test) { TEST(testCase, smlGetTimestampLen_Test) {
// char msg[256] = {0}; uint8_t len = smlGetTimestampLen(0);
// SSmlMsgBuf msgBuf; ASSERT_EQ(len, 1);
// msgBuf.buf = msg;
// msgBuf.len = 256; len = smlGetTimestampLen(1);
// SSmlLineInfo elements = {0}; ASSERT_EQ(len, 1);
//
// SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); len = smlGetTimestampLen(10);
// if(!info){ ASSERT_EQ(len, 2);
// return (TAOS_RES*)request;
// } len = smlGetTimestampLen(390);
// ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); ASSERT_EQ(len, 3);
// if(ret != TSDB_CODE_SUCCESS){
// uError("SML:0x%"PRIx64" smlParseTS failed", info->id); len = smlGetTimestampLen(-1);
// return ret; ASSERT_EQ(len, 1);
// }
len = smlGetTimestampLen(-10);
ASSERT_EQ(len, 2);
len = smlGetTimestampLen(-390);
ASSERT_EQ(len, 3);
}
TEST(testCase, smlProcess_telnet_Test) {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr);
TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db");
taos_free_result(pRes);
pRes = taos_query(taos, "use sml_db");
taos_free_result(pRes);
SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT);
ASSERT_NE(request, nullptr);
SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true);
ASSERT_NE(info, nullptr);
const char *sql[4] = {
"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0",
"sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ",
"sys.if.bytes.out 1479496102 1.3E3 network=tcp",
"sys.procs.running 1479496100 42 host=web01"
};
int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
ASSERT_EQ(ret, 0);
TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a");
ASSERT_NE(res, nullptr);
int fieldNum = taos_field_count(res);
ASSERT_EQ(fieldNum, 2);
int rowNum = taos_affected_rows(res);
ASSERT_EQ(rowNum, 1);
for (int i = 0; i < rowNum; ++i) {
TAOS_ROW rows = taos_fetch_row(res);
}
res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961");
ASSERT_NE(res, nullptr);
fieldNum = taos_field_count(res);
ASSERT_EQ(fieldNum, 2);
rowNum = taos_affected_rows(res);
ASSERT_EQ(rowNum, 2);
for (int i = 0; i < rowNum; ++i) {
TAOS_ROW rows = taos_fetch_row(res);
}
}
TEST(testCase, smlProcess_json_Test) {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr);
TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db");
taos_free_result(pRes);
pRes = taos_query(taos, "use sml_db");
taos_free_result(pRes);
SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT);
ASSERT_NE(request, nullptr);
SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true);
ASSERT_NE(info, nullptr);
const char *sql = "[\n"
" {\n"
" \"metric\": \"sys.cpu.nice\",\n"
" \"timestamp\": 1346846400,\n"
" \"value\": 18,\n"
" \"tags\": {\n"
" \"host\": \"web01\",\n"
" \"dc\": \"lga\"\n"
" }\n"
" },\n"
" {\n"
" \"metric\": \"sys.cpu.nice\",\n"
" \"timestamp\": 1346846400,\n"
" \"value\": 9,\n"
" \"tags\": {\n"
" \"host\": \"web02\",\n"
" \"dc\": \"lga\"\n"
" }\n"
" }\n"
"]";
int ret = smlProcess(info, (char**)(&sql), -1);
ASSERT_EQ(ret, 0);
TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7");
ASSERT_NE(res, nullptr);
int fieldNum = taos_field_count(res);
ASSERT_EQ(fieldNum, 2);
// int rowNum = taos_affected_rows(res);
// ASSERT_EQ(rowNum, 1);
// for (int i = 0; i < rowNum; ++i) {
// TAOS_ROW rows = taos_fetch_row(res);
// } // }
sql = "{\n"
" \"metric\": \"meter_current\",\n"
" \"timestamp\": {\n"
" \"value\" : 1346846400,\n"
" \"type\" : \"s\"\n"
" },\n"
" \"value\": {\n"
" \"value\" : 10.3,\n"
" \"type\" : \"i64\"\n"
" },\n"
" \"tags\": {\n"
" \"groupid\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"bigint\"\n"
" },\n"
" \"location\": { \n"
" \"value\" : \"北京\",\n"
" \"type\" : \"binary\"\n"
" },\n"
" \"id\": \"d1001\"\n"
" }\n"
"}";
ret = smlProcess(info, (char**)(&sql), -1);
ASSERT_EQ(ret, 0);
sql = "{\n"
" \"metric\": \"meter_current\",\n"
" \"timestamp\": {\n"
" \"value\" : 1346846400,\n"
" \"type\" : \"s\"\n"
" },\n"
" \"value\": {\n"
" \"value\" : 10.3,\n"
" \"type\" : \"i64\"\n"
" },\n"
" \"tags\": {\n"
" \"t1\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"bigint\"\n"
" },\n"
" \"t2\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"int\"\n"
" },\n"
" \"t3\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"i16\"\n"
" },\n"
" \"t4\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"i8\"\n"
" },\n"
" \"t5\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"f32\"\n"
" },\n"
" \"t6\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"double\"\n"
" },\n"
" \"t7\": { \n"
" \"value\" : \"8323\",\n"
" \"type\" : \"binary\"\n"
" },\n"
" \"t8\": { \n"
" \"value\" : \"北京\",\n"
" \"type\" : \"binary\"\n"
" },\n"
" \"t9\": { \n"
" \"value\" : true,\n"
" \"type\" : \"bool\"\n"
" },\n"
" \"id\": \"d1001\"\n"
" }\n"
"}";
ret = smlProcess(info, (char**)(&sql), -1);
ASSERT_EQ(ret, 0);
sql = "{\n"
" \"metric\": \"meter_current\",\n"
" \"timestamp\": {\n"
" \"value\" : 1346846400000,\n"
" \"type\" : \"ms\"\n"
" },\n"
" \"value\": \"ni\",\n"
" \"tags\": {\n"
" \"t1\": { \n"
" \"value\" : 20,\n"
" \"type\" : \"i64\"\n"
" },\n"
" \"t2\": { \n"
" \"value\" : 25,\n"
" \"type\" : \"i32\"\n"
" },\n"
" \"t3\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"smallint\"\n"
" },\n"
" \"t4\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"tinyint\"\n"
" },\n"
" \"t5\": { \n"
" \"value\" : 2,\n"
" \"type\" : \"float\"\n"
" },\n"
" \"t6\": { \n"
" \"value\" : 0.2,\n"
" \"type\" : \"f64\"\n"
" },\n"
" \"t7\": \"nsj\",\n"
" \"t8\": { \n"
" \"value\" : \"北京\",\n"
" \"type\" : \"binary\"\n"
" },\n"
" \"t9\": false,\n"
" \"id\": \"d1001\"\n"
" }\n"
"}";
ret = smlProcess(info, (char**)(&sql), -1);
ASSERT_EQ(ret, 0);
}
source/common/src/tdatablock.c
浏览文件 @ eb016e01
...@@ -1447,6 +1447,10 @@ void blockDebugShowData(const SArray* dataBlocks) { ...@@ -1447,6 +1447,10 @@ void blockDebugShowData(const SArray* dataBlocks) {
for (int32_t k = 0; k < colNum; k++) { for (int32_t k = 0; k < colNum; k++) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
if (pColInfoData->hasNull) {
printf(" %15s |", "NULL");
continue;
}
switch (pColInfoData->info.type) { switch (pColInfoData->info.type) {
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
...@@ -1464,6 +1468,9 @@ void blockDebugShowData(const SArray* dataBlocks) { ...@@ -1464,6 +1468,9 @@ void blockDebugShowData(const SArray* dataBlocks) {
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
printf(" %15lu |", *(uint64_t*)var); printf(" %15lu |", *(uint64_t*)var);
break; break;
case TSDB_DATA_TYPE_DOUBLE:
printf(" %15f |", *(double*)var);
break;
} }
} }
printf("\n"); printf("\n");
......
source/dnode/mnode/impl/src/mndTopic.c
浏览文件 @ eb016e01
...@@ -485,8 +485,10 @@ static int32_t mndProcessDropTopicReq(SNodeMsg *pReq) { ...@@ -485,8 +485,10 @@ static int32_t mndProcessDropTopicReq(SNodeMsg *pReq) {
return -1; return -1;
} }
} }
// TODO: check ref
int32_t code = mndDropTopic(pMnode, pReq, pTopic); int32_t code = mndDropTopic(pMnode, pReq, pTopic);
// TODO: iterate and drop related subscriptions and offsets
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
if (code != 0) { if (code != 0) {
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ eb016e01
...@@ -115,6 +115,7 @@ void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond) ...@@ -115,6 +115,7 @@ void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond)
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
int32_t tsdbGetTableGroupFromIdList(SVnode *pVnode, SArray *pTableIdList, STableGroupInfo *pGroupInfo); int32_t tsdbGetTableGroupFromIdList(SVnode *pVnode, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
// tq // tq
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ eb016e01
...@@ -99,7 +99,6 @@ int32_t tsdbInitSma(STsdb *pTsdb); ...@@ -99,7 +99,6 @@ int32_t tsdbInitSma(STsdb *pTsdb);
int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg); int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
typedef enum { typedef enum {
TSDB_FILE_HEAD = 0, // .head TSDB_FILE_HEAD = 0, // .head
TSDB_FILE_DATA, // .data TSDB_FILE_DATA, // .data
......
source/dnode/vnode/src/tq/tq.c
浏览文件 @ eb016e01
...@@ -457,9 +457,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { ...@@ -457,9 +457,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
} }
if (pHeadWithCkSum->head.msgType != TDMT_VND_SUBMIT) { if (pHeadWithCkSum->head.msgType != TDMT_VND_SUBMIT) {
walSkipFetchBody(pExec->pWalReader, pHeadWithCkSum); ASSERT(walSkipFetchBody(pExec->pWalReader, pHeadWithCkSum) == 0);
} else { } else {
walFetchBody(pExec->pWalReader, &pHeadWithCkSum); ASSERT(walFetchBody(pExec->pWalReader, &pHeadWithCkSum) == 0);
} }
SWalReadHead* pHead = &pHeadWithCkSum->head; SWalReadHead* pHead = &pHeadWithCkSum->head;
...@@ -950,6 +950,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { ...@@ -950,6 +950,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
.reader = pStreamReader, .reader = pStreamReader,
.meta = pTq->pVnode->pMeta, .meta = pTq->pVnode->pMeta,
.pMsgCb = &pTq->pVnode->msgCb, .pMsgCb = &pTq->pVnode->msgCb,
.vnode = pTq->pVnode,
}; };
pTask->exec.runners[i].inputHandle = pStreamReader; pTask->exec.runners[i].inputHandle = pStreamReader;
pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
......
source/libs/executor/inc/executorimpl.h
浏览文件 @ eb016e01
...@@ -333,6 +333,8 @@ typedef struct SScanInfo { ...@@ -333,6 +333,8 @@ typedef struct SScanInfo {
typedef struct STableScanInfo { typedef struct STableScanInfo {
void* dataReader; void* dataReader;
SReadHandle readHandle;
SFileBlockLoadRecorder readRecorder; SFileBlockLoadRecorder readRecorder;
int64_t numOfRows; int64_t numOfRows;
int64_t elapsedTime; int64_t elapsedTime;
...@@ -348,6 +350,11 @@ typedef struct STableScanInfo { ...@@ -348,6 +350,11 @@ typedef struct STableScanInfo {
SArray* pColMatchInfo; SArray* pColMatchInfo;
int32_t numOfOutput; int32_t numOfOutput;
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
SqlFunctionCtx* pPseudoCtx;
// int32_t* rowCellInfoOffset;
SQueryTableDataCond cond; SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag; int32_t dataBlockLoadFlag;
...@@ -364,9 +371,18 @@ typedef struct STagScanInfo { ...@@ -364,9 +371,18 @@ typedef struct STagScanInfo {
STableGroupInfo *pTableGroups; STableGroupInfo *pTableGroups;
} STagScanInfo; } STagScanInfo;
typedef enum EStreamScanMode {
STREAM_SCAN_FROM_READERHANDLE = 1,
STREAM_SCAN_FROM_RES,
STREAM_SCAN_FROM_UPDATERES,
STREAM_SCAN_FROM_DATAREADER,
} EStreamScanMode;
typedef struct SStreamBlockScanInfo { typedef struct SStreamBlockScanInfo {
SArray* pBlockLists; // multiple SSDatablock. SArray* pBlockLists; // multiple SSDatablock.
SSDataBlock* pRes; // result SSDataBlock SSDataBlock* pRes; // result SSDataBlock
SSDataBlock* pUpdateRes; // update SSDataBlock
int32_t updateResIndex;
int32_t blockType; // current block type int32_t blockType; // current block type
int32_t validBlockIndex; // Is current data has returned? int32_t validBlockIndex; // Is current data has returned?
SColumnInfo* pCols; // the output column info SColumnInfo* pCols; // the output column info
...@@ -378,6 +394,10 @@ typedef struct SStreamBlockScanInfo { ...@@ -378,6 +394,10 @@ typedef struct SStreamBlockScanInfo {
SArray* tsArray; SArray* tsArray;
SUpdateInfo* pUpdateInfo; SUpdateInfo* pUpdateInfo;
int32_t primaryTsIndex; // primary time stamp slot id int32_t primaryTsIndex; // primary time stamp slot id
void* pDataReader;
EStreamScanMode scanMode;
SOperatorInfo* pOperatorDumy;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
} SStreamBlockScanInfo; } SStreamBlockScanInfo;
typedef struct SSysTableScanInfo { typedef struct SSysTableScanInfo {
...@@ -628,7 +648,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI ...@@ -628,7 +648,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
SArray* pColList); SArray* pColList);
void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win); void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win);
int32_t getTableScanOrder(SOperatorInfo* pOperator); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag);
void doSetOperatorCompleted(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock);
...@@ -644,12 +664,17 @@ SSDataBlock* loadNextDataBlock(void* param); ...@@ -644,12 +664,17 @@ SSDataBlock* loadNextDataBlock(void* param);
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
int32_t type);
SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs);
SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode);
SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo,
char* pData, int16_t bytes, bool masterscan, uint64_t groupId, char* pData, int16_t bytes, bool masterscan, uint64_t groupId,
SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup);
SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, int32_t dataLoadFlag, const uint8_t* scanInfo, SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
...@@ -678,8 +703,9 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx ...@@ -678,8 +703,9 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo,
const STableGroupInfo* pTableGroupInfo); const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataReader, SSDataBlock* pResBlock,
SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pConditions); SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo,
SNode* pConditions, SOperatorInfo* pOperatorDumy, SInterval* pInterval);
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal, SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal,
...@@ -704,7 +730,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim ...@@ -704,7 +730,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList); int32_t numOfOutput, SArray* pPseudoList);
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol);
void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput);
...@@ -733,6 +759,15 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi ...@@ -733,6 +759,15 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi
int32_t length); int32_t length);
void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasicInfo* pInfo, char** result, void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasicInfo* pInfo, char** result,
int32_t* length); int32_t* length);
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts,
SInterval* pInterval, int32_t precision, STimeWindow* win);
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos,
TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item,
int32_t order);
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes,
uint64_t groupId, int32_t numOfOutput);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/libs/executor/src/executorimpl.c
浏览文件 @ eb016e01
...@@ -344,6 +344,28 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, ...@@ -344,6 +344,28 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId,
return pResultRow; return pResultRow;
} }
void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes,
uint64_t groupId, int32_t numOfOutput) {
SAggSupporter* pSup = &pInfo->aggSup;
SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
SResultRowPosition* p1 =
(SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
GET_RES_WINDOW_KEY_LEN(bytes));
SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1);
SqlFunctionCtx* pCtx = pInfo->binfo.pCtx;
for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultCell(pResult, i, pInfo->binfo.rowCellInfoOffset);
struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
continue;
}
pResInfo->initialized = false;
if (pCtx[i].functionId != -1) {
pCtx[i].fpSet.init(&pCtx[i], pResInfo);
}
}
}
/** /**
* the struct of key in hash table * the struct of key in hash table
* +----------+---------------+ * +----------+---------------+
...@@ -654,7 +676,7 @@ static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* p ...@@ -654,7 +676,7 @@ static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* p
} }
static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order,
bool createDummyCol); int32_t scanFlag, bool createDummyCol);
static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock,
int32_t order) { int32_t order) {
...@@ -665,12 +687,12 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC ...@@ -665,12 +687,12 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC
} }
} }
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol) { bool createDummyCol) {
if (pBlock->pBlockAgg != NULL) { if (pBlock->pBlockAgg != NULL) {
doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order);
} else { } else {
doSetInputDataBlock(pOperator, pCtx, pBlock, order, createDummyCol); doSetInputDataBlock(pOperator, pCtx, pBlock, order, scanFlag, createDummyCol);
} }
} }
...@@ -717,14 +739,14 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc ...@@ -717,14 +739,14 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc
} }
static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order,
bool createDummyCol) { int32_t scanFlag, bool createDummyCol) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { for (int32_t i = 0; i < pOperator->numOfExprs; ++i) {
pCtx[i].order = order; pCtx[i].order = order;
pCtx[i].size = pBlock->info.rows; pCtx[i].size = pBlock->info.rows;
pCtx[i].pSrcBlock = pBlock; pCtx[i].pSrcBlock = pBlock;
pCtx[i].currentStage = MAIN_SCAN; pCtx[i].currentStage = scanFlag;
SInputColumnInfoData* pInput = &pCtx[i].input; SInputColumnInfoData* pInput = &pCtx[i].input;
pInput->uid = pBlock->info.uid; pInput->uid = pBlock->info.uid;
...@@ -740,7 +762,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt ...@@ -740,7 +762,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
pInput->numOfRows = pBlock->info.rows; pInput->numOfRows = pBlock->info.rows;
pInput->startRowIndex = 0; pInput->startRowIndex = 0;
// the last parameter is the timestamp column // NOTE: the last parameter is the primary timestamp column
if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) {
pInput->pPTS = pInput->pData[j]; pInput->pPTS = pInput->pData[j];
} }
...@@ -884,7 +906,8 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc ...@@ -884,7 +906,8 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) { } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) {
ASSERT(!fmIsAggFunc(pfCtx->functionId)); ASSERT(!fmIsAggFunc(pfCtx->functionId));
if (fmIsPseudoColumnFunc(pfCtx->functionId)) { // _rowts/_c0, not tbname column
if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) {
// do nothing // do nothing
} else if (fmIsNonstandardSQLFunc(pfCtx->functionId)) { } else if (fmIsNonstandardSQLFunc(pfCtx->functionId)) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]); SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]);
...@@ -3506,7 +3529,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { ...@@ -3506,7 +3529,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) {
break; break;
} }
setInputDataBlock(pOperator, pInfo->binfo.pCtx, pDataBlock, TSDB_ORDER_ASC, true); setInputDataBlock(pOperator, pInfo->binfo.pCtx, pDataBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
// updateOutputBuf(&pInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor, // updateOutputBuf(&pInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor,
// pOperator->pRuntimeEnv, true); // pOperator->pRuntimeEnv, true);
doMergeImpl(pOperator, pOperator->numOfExprs, pDataBlock); doMergeImpl(pOperator, pOperator->numOfExprs, pDataBlock);
...@@ -3671,17 +3694,24 @@ _error: ...@@ -3671,17 +3694,24 @@ _error:
return NULL; return NULL;
} }
int32_t getTableScanOrder(SOperatorInfo* pOperator) { int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag) {
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { // todo add more information about exchange operation
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) {
*order = TSDB_ORDER_ASC;
*scanFlag = MAIN_SCAN;
return TSDB_CODE_SUCCESS;
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pTableScanInfo = pOperator->info;
*order = pTableScanInfo->cond.order;
*scanFlag = pTableScanInfo->scanFlag;
return TSDB_CODE_SUCCESS;
} else {
if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) {
return TSDB_ORDER_ASC; return TSDB_CODE_INVALID_PARA;
} else { } else {
return getTableScanOrder(pOperator->pDownstream[0]); return getTableScanInfo(pOperator->pDownstream[0], order, scanFlag);
} }
} }
STableScanInfo* pTableScanInfo = pOperator->info;
return pTableScanInfo->cond.order;
} }
// this is a blocking operator // this is a blocking operator
...@@ -3697,6 +3727,9 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { ...@@ -3697,6 +3727,9 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
SOperatorInfo* downstream = pOperator->pDownstream[0]; SOperatorInfo* downstream = pOperator->pDownstream[0];
int32_t order = TSDB_ORDER_ASC;
int32_t scanFlag = MAIN_SCAN;
while (1) { while (1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
...@@ -3709,11 +3742,14 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { ...@@ -3709,11 +3742,14 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
// setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfExprs);
// } // }
int32_t order = getTableScanOrder(pOperator); int32_t code = getTableScanInfo(pOperator, &order, &scanFlag);
if (code != TSDB_CODE_SUCCESS) {
longjmp(pTaskInfo->env, code);
}
// there is an scalar expression that needs to be calculated before apply the group aggregation. // there is an scalar expression that needs to be calculated before apply the group aggregation.
if (pAggInfo->pScalarExprInfo != NULL) { if (pAggInfo->pScalarExprInfo != NULL) {
int32_t code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx,
pAggInfo->numOfScalarExpr, NULL); pAggInfo->numOfScalarExpr, NULL);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code; pTaskInfo->code = code;
...@@ -3723,7 +3759,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { ...@@ -3723,7 +3759,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setExecutionContext(pOperator->numOfExprs, pBlock->info.groupId, pTaskInfo, pAggInfo); setExecutionContext(pOperator->numOfExprs, pBlock->info.groupId, pTaskInfo, pAggInfo);
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, true); setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, true);
doAggregateImpl(pOperator, 0, pInfo->pCtx); doAggregateImpl(pOperator, 0, pInfo->pCtx);
#if 0 // test for encode/decode result info #if 0 // test for encode/decode result info
...@@ -4004,6 +4040,9 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { ...@@ -4004,6 +4040,9 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
} }
#endif #endif
int32_t order = 0;
int32_t scanFlag = 0;
SOperatorInfo* downstream = pOperator->pDownstream[0]; SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) { while (1) {
...@@ -4035,15 +4074,14 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { ...@@ -4035,15 +4074,14 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
// } // }
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
int32_t order = getTableScanOrder(pOperator->pDownstream[0]); int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, false); setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false);
blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
pTaskInfo->code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, pProjectInfo->pPseudoColInfo);
pProjectInfo->pPseudoColInfo); if (code != TSDB_CODE_SUCCESS) {
if (pTaskInfo->code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code);
longjmp(pTaskInfo->env, pTaskInfo->code);
} }
int32_t status = handleLimitOffset(pOperator, pBlock); int32_t status = handleLimitOffset(pOperator, pBlock);
...@@ -4642,8 +4680,22 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* ...@@ -4642,8 +4680,22 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
pExp->pExpr->_function.functionId = pFuncNode->funcId; pExp->pExpr->_function.functionId = pFuncNode->funcId;
pExp->pExpr->_function.pFunctNode = pFuncNode; pExp->pExpr->_function.pFunctNode = pFuncNode;
strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
tListLen(pExp->pExpr->_function.functionName)); tListLen(pExp->pExpr->_function.functionName));
#if 1
// todo refactor: add the parameter for tbname function
if (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0) {
pFuncNode->pParameterList = nodesMakeList();
ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == res) { // todo handle error
} else {
res->node.resType = (SDataType) {.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
nodesListAppend(pFuncNode->pParameterList, res);
}
}
#endif
int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList);
...@@ -4704,58 +4756,29 @@ static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t ...@@ -4704,58 +4756,29 @@ static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t
uint64_t queryId, uint64_t taskId); uint64_t queryId, uint64_t taskId);
static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo);
static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* extractColumnInfo(SNodeList* pNodeList);
static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
int32_t type);
static SArray* createSortInfo(SNodeList* pNodeList); static SArray* createSortInfo(SNodeList* pNodeList);
static SArray* extractPartitionColInfo(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList);
static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode);
static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode);
static SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) {
SInterval interval = {
.interval = pTableScanNode->interval,
.sliding = pTableScanNode->sliding,
.intervalUnit = pTableScanNode->intervalUnit,
.slidingUnit = pTableScanNode->slidingUnit,
.offset = pTableScanNode->offset,
};
return interval;
}
SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) {
int32_t type = nodeType(pPhyNode); int32_t type = nodeType(pPhyNode);
if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode;
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
int32_t numOfCols = 0;
tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId);
if (pDataReader == NULL && terrno != 0) { if (pDataReader == NULL && terrno != 0) {
return NULL; return NULL;
} }
SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
SSDataBlock* pResBlock = createResDataBlock(pDescNode);
SQueryTableDataCond cond = {0};
int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
SInterval interval = extractIntervalInfo(pTableScanNode);
SOperatorInfo* pOperator = createTableScanOperatorInfo(
pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList, pResBlock,
pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
STableScanInfo* pScanInfo = pOperator->info; STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator; return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode;
...@@ -4763,18 +4786,48 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo ...@@ -4763,18 +4786,48 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo); return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, int32_t numOfCols = 0;
tsdbReaderT pDataReader = NULL;
if (pHandle->vnode) {
pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId);
} else {
doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo,
queryId, taskId); queryId, taskId);
SArray* tableIdList = extractTableIdList(pTableGroupInfo); }
if (pDataReader == NULL && terrno != 0) {
qDebug("pDataReader is NULL");
// return NULL;
} else {
qDebug("pDataReader is not NULL");
}
SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
SSDataBlock* pResBlockDumy = createResDataBlock(pDescNode);
SQueryTableDataCond cond = {0};
int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
SInterval interval = extractIntervalInfo(pTableScanNode);
SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(
pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList,
pResBlockDumy, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
// int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo,
// queryId, taskId);
SArray* tableIdList = extractTableIdList(pTableGroupInfo);
SSDataBlock* pResBlock = createResDataBlock(pDescNode); SSDataBlock* pResBlock = createResDataBlock(pDescNode);
int32_t numOfCols = 0;
SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo, SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pDataReader, pResBlock, pCols, tableIdList, pTaskInfo,
pScanPhyNode->node.pConditions); pScanPhyNode->node.pConditions, pOperatorDumy, &interval);
taosArrayDestroy(tableIdList); taosArrayDestroy(tableIdList);
return pOperator; return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
...@@ -4945,7 +4998,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo ...@@ -4945,7 +4998,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return pOptr; return pOptr;
} }
static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
pCond->loadExternalRows = false; pCond->loadExternalRows = false;
pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
......
source/libs/executor/src/groupoperator.c
浏览文件 @ eb016e01
...@@ -287,7 +287,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { ...@@ -287,7 +287,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
} }
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
// there is an scalar expression that needs to be calculated right before apply the group aggregation. // there is an scalar expression that needs to be calculated right before apply the group aggregation.
if (pInfo->pScalarExprInfo != NULL) { if (pInfo->pScalarExprInfo != NULL) {
......
source/libs/executor/src/scanoperator.c
浏览文件 @ eb016e01
...@@ -13,7 +13,6 @@ ...@@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <libs/function/function.h>
#include "filter.h" #include "filter.h"
#include "function.h" #include "function.h"
#include "functionMgt.h" #include "functionMgt.h"
...@@ -284,6 +283,27 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { ...@@ -284,6 +283,27 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
continue; continue;
} }
// currently only the tbname pseudo column
if (pTableScanInfo->numOfPseudoExpr > 0) {
int32_t dstSlotId = pTableScanInfo->pPseudoExpr->base.resSchema.slotId;
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId);
colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows);
struct SScalarFuncExecFuncs fpSet;
fmGetScalarFuncExecFuncs(pTableScanInfo->pPseudoExpr->pExpr->_function.functionId, &fpSet);
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_BIGINT;
infoData.info.bytes = sizeof(uint64_t);
colInfoDataEnsureCapacity(&infoData, 0, 1);
colDataAppendInt64(&infoData, 0, &pBlock->info.uid);
SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pTableScanInfo->readHandle.meta, .columnData = &infoData};
SScalarParam param = {.columnData = pColInfoData};
fpSet.process(&srcParam, 1, &param);
}
return pBlock; return pBlock;
} }
...@@ -314,8 +334,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { ...@@ -314,8 +334,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STimeWindow* pWin = &pTableScanInfo->cond.twindow; STimeWindow* pWin = &pTableScanInfo->cond.twindow;
qDebug("%s start to repeat ascending order scan data blocks due to query func required, qrange:%" PRId64 qDebug("%s start to repeat ascending order scan data blocks due to query func required, qrange:%" PRId64
"-%" PRId64, "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
// do prepare for the next round table scan operation // do prepare for the next round table scan operation
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond);
...@@ -359,10 +378,29 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { ...@@ -359,10 +378,29 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
return NULL; return NULL;
} }
SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) {
int32_t dataLoadFlag, const uint8_t* scanInfo, SArray* pColMatchInfo, SInterval interval = {
SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, .interval = pTableScanNode->interval,
double sampleRatio, SExecTaskInfo* pTaskInfo) { .sliding = pTableScanNode->sliding,
.intervalUnit = pTableScanNode->intervalUnit,
.slidingUnit = pTableScanNode->slidingUnit,
.offset = pTableScanNode->offset,
};
return interval;
}
static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) {
STableScanInfo* pTableScanInfo = (STableScanInfo*)param;
taosMemoryFree(pTableScanInfo->pResBlock);
tsdbCleanupReadHandle(pTableScanInfo->dataReader);
if (pTableScanInfo->pColMatchInfo != NULL) {
taosArrayDestroy(pTableScanInfo->pColMatchInfo);
}
}
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) {
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
...@@ -373,27 +411,42 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCon ...@@ -373,27 +411,42 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCon
return NULL; return NULL;
} }
pInfo->cond = *pCond; SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
pInfo->scanInfo = (SScanInfo){.numOfAsc = scanInfo[0], .numOfDesc = scanInfo[1]};
pInfo->interval = *pInterval; int32_t numOfCols = 0;
pInfo->sampleRatio = sampleRatio; SArray* pColList = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
pInfo->dataBlockLoadFlag = dataLoadFlag;
pInfo->pResBlock = pResBlock; int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
pInfo->pFilterNode = pCondition; if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
if (pTableScanNode->scan.pScanPseudoCols != NULL) {
pInfo->pPseudoExpr = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pInfo->numOfPseudoExpr);
pInfo->pPseudoCtx = createSqlFunctionCtx(pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, &pInfo->rowCellInfoOffset);
}
pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]};
pInfo->readHandle = *readHandle;
pInfo->interval = extractIntervalInfo(pTableScanNode);
pInfo->sampleRatio = pTableScanNode->ratio;
pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired;
pInfo->pResBlock = createResDataBlock(pDescNode);
pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
pInfo->dataReader = pDataReader; pInfo->dataReader = pDataReader;
pInfo->scanFlag = MAIN_SCAN; pInfo->scanFlag = MAIN_SCAN;
pInfo->pColMatchInfo = pColMatchInfo; pInfo->pColMatchInfo = pColList;
pOperator->name = "TableScanOperator"; // for dubug purpose pOperator->name = "TableScanOperator"; // for debug purpose
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
pOperator->blocking = false; pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED; pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo; pOperator->info = pInfo;
pOperator->numOfExprs = numOfOutput; pOperator->numOfExprs = numOfCols;
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, NULL, NULL, NULL, NULL); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, NULL, NULL, NULL);
static int32_t cost = 0; static int32_t cost = 0;
...@@ -515,7 +568,40 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { ...@@ -515,7 +568,40 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) {
taosArrayClear(pInfo->pBlockLists); taosArrayClear(pInfo->pBlockLists);
} }
static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo) { static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
SSDataBlock* pSDB = pInfo->pUpdateRes;
if (pInfo->updateResIndex < pSDB->info.rows) {
SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, 0);
TSKEY *tsCols = (TSKEY*)pColDataInfo->pData;
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[pInfo->updateResIndex], &pInfo->interval,
pInfo->interval.precision, NULL);
STableScanInfo* pTableScanInfo = pInfo->pOperatorDumy->info;
pTableScanInfo->cond.twindow = win;
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond);
pInfo->updateResIndex += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, pInfo->updateResIndex,
win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
pTableScanInfo->scanTimes = 0;
return true;
} else {
return false;
}
}
static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) {
SSDataBlock* pResult = NULL;
pResult = doTableScan(pInfo->pOperatorDumy);
if (pResult == NULL) {
if (prepareDataScan(pInfo)) {
// scan next window data
pResult = doTableScan(pInfo->pOperatorDumy);
}
}
return pResult;
}
static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool invertible) {
SColumnInfoData* pColDataInfo = taosArrayGet(pInfo->pRes->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pColDataInfo = taosArrayGet(pInfo->pRes->pDataBlock, pInfo->primaryTsIndex);
TSKEY* ts = (TSKEY*)pColDataInfo->pData; TSKEY* ts = (TSKEY*)pColDataInfo->pData;
for (int32_t i = 0; i < pInfo->pRes->info.rows; i++) { for (int32_t i = 0; i < pInfo->pRes->info.rows; i++) {
...@@ -523,13 +609,19 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo) { ...@@ -523,13 +609,19 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo) {
taosArrayPush(pInfo->tsArray, ts + i); taosArrayPush(pInfo->tsArray, ts + i);
} }
} }
if (taosArrayGetSize(pInfo->tsArray) > 0) { int32_t size = taosArrayGetSize(pInfo->tsArray);
if (size > 0 && invertible) {
// TODO(liuyao) get from tsdb // TODO(liuyao) get from tsdb
// SSDataBlock* p = createOneDataBlock(pInfo->pRes, true); // SSDataBlock* p = createOneDataBlock(pInfo->pRes, true);
// p->info.type = STREAM_INVERT; // p->info.type = STREAM_INVERT;
// taosArrayClear(pInfo->tsArray); // taosArrayClear(pInfo->tsArray);
// return p; // return p;
return NULL; SSDataBlock* p = createOneDataBlock(pInfo->pRes, false);
taosArraySet(p->pDataBlock, 0, pInfo->tsArray);
p->info.rows = size;
p->info.type = STREAM_REPROCESS;
taosArrayClear(pInfo->tsArray);
return p;
} }
return NULL; return NULL;
} }
...@@ -556,14 +648,23 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { ...@@ -556,14 +648,23 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
int32_t current = pInfo->validBlockIndex++; int32_t current = pInfo->validBlockIndex++;
return taosArrayGetP(pInfo->pBlockLists, current); return taosArrayGetP(pInfo->pBlockLists, current);
} else { } else {
if (total > 0) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
ASSERT(total == 2); blockDataDestroy(pInfo->pUpdateRes);
SSDataBlock* pRes = taosArrayGetP(pInfo->pBlockLists, 0); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
SSDataBlock* pUpRes = taosArrayGetP(pInfo->pBlockLists, 1); return pInfo->pRes;
blockDataDestroy(pUpRes); } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) {
taosArrayClear(pInfo->pBlockLists); blockDataCleanup(pInfo->pRes);
return pRes; pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
return pInfo->pUpdateRes;
} else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
SSDataBlock* pSDB = doDataScan(pInfo);
if (pSDB == NULL) {
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
} else {
return pSDB;
} }
}
SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
blockDataCleanup(pInfo->pRes); blockDataCleanup(pInfo->pRes);
...@@ -629,21 +730,29 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { ...@@ -629,21 +730,29 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
if (rows == 0) { if (rows == 0) {
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
} else { } else if (pInfo->interval.interval > 0) {
SSDataBlock* upRes = getUpdateDataBlock(pInfo); SSDataBlock* upRes = getUpdateDataBlock(pInfo, true); //TODO(liuyao) get invertible from plan
if (upRes) { if (upRes) {
taosArrayPush(pInfo->pBlockLists, &(pInfo->pRes)); pInfo->pUpdateRes = upRes;
taosArrayPush(pInfo->pBlockLists, &upRes); if (upRes->info.type = STREAM_REPROCESS) {
pInfo->updateResIndex = 0;
prepareDataScan(pInfo);
pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
} else if (upRes->info.type = STREAM_INVERT) {
pInfo->scanMode = STREAM_SCAN_FROM_RES;
return upRes; return upRes;
} }
} }
}
return (rows == 0) ? NULL : pInfo->pRes; return (rows == 0) ? NULL : pInfo->pRes;
} }
} }
SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataReader,
SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pCondition) { SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList,
SExecTaskInfo* pTaskInfo, SNode* pCondition, SOperatorInfo* pOperatorDumy,
SInterval* pInterval) {
SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
...@@ -683,7 +792,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* ...@@ -683,7 +792,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
} }
pInfo->primaryTsIndex = 0; // TODO(liuyao) get it from physical plan pInfo->primaryTsIndex = 0; // TODO(liuyao) get it from physical plan
pInfo->pUpdateInfo = updateInfoInit(60000, 0, 100); // TODO(liuyao) get it from physical plan pInfo->pUpdateInfo = updateInfoInitP(pInterval, 10000); // TODO(liuyao) get watermark from physical plan
if (pInfo->pUpdateInfo == NULL) { if (pInfo->pUpdateInfo == NULL) {
taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
...@@ -693,6 +802,10 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* ...@@ -693,6 +802,10 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
pInfo->readerHandle = streamReadHandle; pInfo->readerHandle = streamReadHandle;
pInfo->pRes = pResBlock; pInfo->pRes = pResBlock;
pInfo->pCondition = pCondition; pInfo->pCondition = pCondition;
pInfo->pDataReader = pDataReader;
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
pInfo->pOperatorDumy = pOperatorDumy;
pInfo->interval = *pInterval;
pOperator->name = "StreamBlockScanOperator"; pOperator->name = "StreamBlockScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
...@@ -1311,7 +1424,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { ...@@ -1311,7 +1424,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
metaReaderClear(&mr); metaReaderClear(&mr);
colDataAppend(pDst, count, str, false); colDataAppend(pDst, count, str, false);
// data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); // data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
// dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; // dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
// doSetTagValueToResultBuf(dst, data, type, bytes); // doSetTagValueToResultBuf(dst, data, type, bytes);
......
source/libs/executor/src/timewindowoperator.c
浏览文件 @ eb016e01
...@@ -82,7 +82,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T ...@@ -82,7 +82,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T
} }
// get the correct time window according to the handled timestamp // get the correct time window according to the handled timestamp
static STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts,
SInterval* pInterval, int32_t precision, STimeWindow* win) { SInterval* pInterval, int32_t precision, STimeWindow* win) {
STimeWindow w = {0}; STimeWindow w = {0};
...@@ -186,7 +186,7 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se ...@@ -186,7 +186,7 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se
return forwardStep; return forwardStep;
} }
static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
int32_t midPos = -1; int32_t midPos = -1;
int32_t numOfRows; int32_t numOfRows;
...@@ -249,7 +249,7 @@ static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { ...@@ -249,7 +249,7 @@ static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
return midPos; return midPos;
} }
static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos,
TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item,
int32_t order) { int32_t order) {
assert(startPos >= 0 && startPos < pDataBlockInfo->rows); assert(startPos >= 0 && startPos < pDataBlockInfo->rows);
...@@ -775,7 +775,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { ...@@ -775,7 +775,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
// setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs);
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; STableQueryInfo* pTableQueryInfo = pInfo->pCurrent;
setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window);
...@@ -910,7 +910,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { ...@@ -910,7 +910,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
break; break;
} }
setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, MAIN_SCAN, true);
doStateWindowAggImpl(pOperator, pInfo, pBlock); doStateWindowAggImpl(pOperator, pInfo, pBlock);
} }
...@@ -988,6 +988,20 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type ...@@ -988,6 +988,20 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type
} }
} }
} }
static void doClearWindows(SIntervalAggOperatorInfo* pInfo, int32_t numOfOutput, SSDataBlock* pBlock) {
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
TSKEY *tsCols = (TSKEY*)pColDataInfo->pData;
int32_t step = 0;
for (int32_t i = 0; i < pBlock->info.rows; i += step) {
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], &pInfo->interval,
pInfo->interval.precision, NULL);
step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i,
win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
doClearWindow(pInfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
}
}
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
SIntervalAggOperatorInfo* pInfo = pOperator->info; SIntervalAggOperatorInfo* pInfo = pOperator->info;
...@@ -1024,10 +1038,14 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { ...@@ -1024,10 +1038,14 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
// setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs);
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
if (pInfo->invertible) { if (pInfo->invertible) {
setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type); setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type);
} }
if (pBlock->info.type == STREAM_REPROCESS) {
doClearWindows(pInfo, pOperator->numOfExprs, pBlock);
continue;
}
pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0);
} }
...@@ -1286,7 +1304,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { ...@@ -1286,7 +1304,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
} }
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, MAIN_SCAN, true);
doSessionWindowAggImpl(pOperator, pInfo, pBlock); doSessionWindowAggImpl(pOperator, pInfo, pBlock);
} }
...@@ -1334,7 +1352,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { ...@@ -1334,7 +1352,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) {
// setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfExprs); // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfExprs);
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true);
// hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0);
} }
......
source/libs/function/inc/builtinsimpl.h
浏览文件 @ eb016e01
...@@ -98,6 +98,10 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx); ...@@ -98,6 +98,10 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx);
bool getCsumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getCsumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t csumFunction(SqlFunctionCtx* pCtx); int32_t csumFunction(SqlFunctionCtx* pCtx);
bool getMavgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool mavgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t mavgFunction(SqlFunctionCtx* pCtx);
bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
#ifdef __cplusplus #ifdef __cplusplus
......
source/libs/function/src/builtins.c
浏览文件 @ eb016e01
...@@ -339,6 +339,27 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { ...@@ -339,6 +339,27 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateMavg(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (2 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
if (QUERY_NODE_COLUMN != nodeType(pPara)) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
"The input parameter of MAVG function can only be column");
}
uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
if (!IS_NUMERIC_TYPE(colType) || !IS_INTEGER_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
return TSDB_CODE_SUCCESS;
}
static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// todo // todo
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -783,6 +804,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -783,6 +804,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.processFunc = csumFunction, .processFunc = csumFunction,
.finalizeFunc = NULL .finalizeFunc = NULL
}, },
{
.name = "mavg",
.type = FUNCTION_TYPE_MAVG,
.classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateMavg,
.getEnvFunc = getMavgFuncEnv,
.initFunc = mavgFunctionSetup,
.processFunc = mavgFunction,
.finalizeFunc = NULL
},
{ {
.name = "abs", .name = "abs",
.type = FUNCTION_TYPE_ABS, .type = FUNCTION_TYPE_ABS,
......
source/libs/function/src/builtinsimpl.c
浏览文件 @ eb016e01
...@@ -21,7 +21,8 @@ ...@@ -21,7 +21,8 @@
#include "tdatablock.h" #include "tdatablock.h"
#include "tpercentile.h" #include "tpercentile.h"
#define HISTOGRAM_MAX_BINS_NUM 100 #define HISTOGRAM_MAX_BINS_NUM 1000
#define MAVG_MAX_POINTS_NUM 1000
typedef struct SSumRes { typedef struct SSumRes {
union { union {
...@@ -141,6 +142,14 @@ typedef enum { ...@@ -141,6 +142,14 @@ typedef enum {
STATE_OPER_EQ, STATE_OPER_EQ,
} EStateOperType; } EStateOperType;
typedef struct SMavgInfo {
int32_t pos;
double sum;
int32_t numOfPoints;
bool pointsMeet;
double points[];
} SMavgInfo;
#define SET_VAL(_info, numOfElem, res) \ #define SET_VAL(_info, numOfElem, res) \
do { \ do { \
if ((numOfElem) <= 0) { \ if ((numOfElem) <= 0) { \
...@@ -1644,7 +1653,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { ...@@ -1644,7 +1653,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) {
pResInfo->complete = true; pResInfo->complete = true;
return 0; return 0;
} else { } else {
pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); pInfo->pMemBucket = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval);
} }
} }
...@@ -1695,10 +1704,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { ...@@ -1695,10 +1704,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) {
pInfo->numOfElems += 1; pInfo->numOfElems += 1;
} }
} }
} else {
return 0;
}
// the second stage, calculate the true percentile value // the second stage, calculate the true percentile value
int32_t start = pInput->startRowIndex; int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < pInput->numOfRows + start; ++i) { for (int32_t i = start; i < pInput->numOfRows + start; ++i) {
...@@ -1707,18 +1713,19 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { ...@@ -1707,18 +1713,19 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) {
} }
char* data = colDataGetData(pCol, i); char* data = colDataGetData(pCol, i);
notNullElems += 1; notNullElems += 1;
tMemBucketPut(pInfo->pMemBucket, data, 1); tMemBucketPut(pInfo->pMemBucket, data, 1);
} }
SET_VAL(pResInfo, notNullElems, 1); SET_VAL(pResInfo, notNullElems, 1);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SVariant* pVal = &pCtx->param[1].param; SVariant* pVal = &pCtx->param[1].param;
double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; double v = (pVal->nType == TSDB_DATA_TYPE_BIGINT) ? pVal->i : pVal->d;
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo* ppInfo = (SPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo); SPercentileInfo* ppInfo = (SPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo);
...@@ -2949,3 +2956,81 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { ...@@ -2949,3 +2956,81 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) {
return numOfElems; return numOfElems;
} }
bool getMavgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SMavgInfo) + MAVG_MAX_POINTS_NUM * sizeof(double);
return true;
}
bool mavgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) {
return false;
}
SMavgInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
pInfo->pos = 0;
pInfo->sum = 0;
pInfo->numOfPoints = pCtx->param[1].param.i;
if (pInfo->numOfPoints < 1 || pInfo->numOfPoints > MAVG_MAX_POINTS_NUM) {
return false;
}
pInfo->pointsMeet = false;
return true;
}
int32_t mavgFunction(SqlFunctionCtx* pCtx) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SMavgInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SInputColumnInfoData* pInput = &pCtx->input;
TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
SColumnInfoData* pInputCol = pInput->pData[0];
SColumnInfoData* pTsOutput = pCtx->pTsOutput;
SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
int32_t numOfElems = 0;
int32_t type = pInputCol->info.type;
int32_t startOffset = pCtx->offset;
for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) {
int32_t pos = startOffset + numOfElems;
if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
//colDataAppendNULL(pOutput, i);
continue;
}
char* data = colDataGetData(pInputCol, i);
double v;
GET_TYPED_DATA(v, double, type, data);
if (!pInfo->pointsMeet && (pInfo->pos < pInfo->numOfPoints - 1)) {
pInfo->points[pInfo->pos] = v;
pInfo->sum += v;
} else {
if (!pInfo->pointsMeet && (pInfo->pos == pInfo->numOfPoints - 1)) {
pInfo->sum +=v;
pInfo->pointsMeet = true;
} else {
pInfo->sum = pInfo->sum + v - pInfo->points[pInfo->pos];
}
pInfo->points[pInfo->pos] = v;
double result = pInfo->sum / pInfo->numOfPoints;
colDataAppend(pOutput, pos, (char *)&result, false);
//TODO: remove this after pTsOutput is handled
if (pTsOutput != NULL) {
colDataAppendInt64(pTsOutput, pos, &tsList[i]);
}
numOfElems++;
}
pInfo->pos++;
if (pInfo->pos == pInfo->numOfPoints) {
pInfo->pos = 0;
}
}
return numOfElems;
}
source/libs/function/src/texpr.c
浏览文件 @ eb016e01
...@@ -27,19 +27,6 @@ ...@@ -27,19 +27,6 @@
#include "tvariant.h" #include "tvariant.h"
#include "tdef.h" #include "tdef.h"
//static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
// if (pLeft->nodeType == TEXPR_COL_NODE) {
// // if left node is the primary column,return true
// return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0;
// } else {
// // if any children have query on primary key, their parents are also keep this value
// return ((pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pLeft->_node.hasPK == 1) ||
// (pRight->nodeType == TEXPR_BINARYEXPR_NODE && pRight->_node.hasPK == 1)) == true
// ? 1
// : 0;
// }
//}
static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)); static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *));
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) {
...@@ -64,21 +51,7 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { ...@@ -64,21 +51,7 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
} }
int32_t type = (*pExpr)->nodeType; int32_t type = (*pExpr)->nodeType;
if (type == TEXPR_BINARYEXPR_NODE) { if (type == TEXPR_VALUE_NODE) {
doExprTreeDestroy(&(*pExpr)->_node.pLeft, fp);
doExprTreeDestroy(&(*pExpr)->_node.pRight, fp);
if (fp != NULL) {
fp((*pExpr)->_node.info);
}
} else if (type == TEXPR_UNARYEXPR_NODE) {
doExprTreeDestroy(&(*pExpr)->_node.pLeft, fp);
if (fp != NULL) {
fp((*pExpr)->_node.info);
}
assert((*pExpr)->_node.pRight == NULL);
} else if (type == TEXPR_VALUE_NODE) {
taosVariantDestroy((*pExpr)->pVal); taosVariantDestroy((*pExpr)->pVal);
taosMemoryFree((*pExpr)->pVal); taosMemoryFree((*pExpr)->pVal);
} else if (type == TEXPR_COL_NODE) { } else if (type == TEXPR_COL_NODE) {
...@@ -90,9 +63,7 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { ...@@ -90,9 +63,7 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
} }
bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) {
tExprNode *pLeft = pExpr->_node.pLeft; #if 0
tExprNode *pRight = pExpr->_node.pRight;
//non-leaf nodes, recursively traverse the expression tree in the post-root order //non-leaf nodes, recursively traverse the expression tree in the post-root order
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) { if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
if (pExpr->_node.optr == LOGIC_COND_TYPE_OR) { // or if (pExpr->_node.optr == LOGIC_COND_TYPE_OR) { // or
...@@ -114,6 +85,9 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp ...@@ -114,6 +85,9 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp
// handle the leaf node // handle the leaf node
param->setupInfoFn(pExpr, param->pExtInfo); param->setupInfoFn(pExpr, param->pExtInfo);
return param->nodeFilterFn(pItem, pExpr->_node.info); return param->nodeFilterFn(pItem, pExpr->_node.info);
#endif
return 0;
} }
// TODO: these three functions should be made global // TODO: these three functions should be made global
...@@ -141,59 +115,6 @@ static UNUSED_FUNC char* exception_strdup(const char* str) { ...@@ -141,59 +115,6 @@ static UNUSED_FUNC char* exception_strdup(const char* str) {
return p; return p;
} }
static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
int32_t anchor = CLEANUP_GET_ANCHOR();
if (CLEANUP_EXCEED_LIMIT()) {
THROW(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT);
return NULL;
}
tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode));
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, pExpr, NULL);
pExpr->nodeType = tbufReadUint8(br);
if (pExpr->nodeType == TEXPR_VALUE_NODE) {
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
pExpr->pVal = pVal;
pVal->nType = tbufReadUint32(br);
if (pVal->nType == TSDB_DATA_TYPE_BINARY) {
tbufReadToBuffer(br, &pVal->nLen, sizeof(pVal->nLen));
pVal->pz = taosMemoryCalloc(1, pVal->nLen + 1);
tbufReadToBuffer(br, pVal->pz, pVal->nLen);
} else {
pVal->i = tbufReadInt64(br);
}
} else if (pExpr->nodeType == TEXPR_COL_NODE) {
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
pExpr->pSchema = pSchema;
pSchema->colId = tbufReadInt16(br);
pSchema->bytes = tbufReadInt16(br);
pSchema->type = tbufReadUint8(br);
tbufReadToString(br, pSchema->name, TSDB_COL_NAME_LEN);
} else if (pExpr->nodeType == TEXPR_BINARYEXPR_NODE) {
pExpr->_node.optr = tbufReadUint8(br);
pExpr->_node.pLeft = exprTreeFromBinaryImpl(br);
pExpr->_node.pRight = exprTreeFromBinaryImpl(br);
assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL);
}
CLEANUP_EXECUTE_TO(anchor, false);
return pExpr;
}
tExprNode* exprTreeFromBinary(const void* data, size_t size) {
if (size == 0) {
return NULL;
}
SBufferReader br = tbufInitReader(data, size, false);
return exprTreeFromBinaryImpl(&br);
}
void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) { void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) {
SBufferReader br = tbufInitReader(buf, len, false); SBufferReader br = tbufInitReader(buf, len, false);
uint32_t type = tbufReadUint32(&br); uint32_t type = tbufReadUint32(&br);
...@@ -405,38 +326,3 @@ err_ret: ...@@ -405,38 +326,3 @@ err_ret:
taosHashCleanup(pObj); taosHashCleanup(pObj);
taosMemoryFreeClear(tmp); taosMemoryFreeClear(tmp);
} }
tExprNode* exprdup(tExprNode* pNode) {
if (pNode == NULL) {
return NULL;
}
tExprNode* pCloned = taosMemoryCalloc(1, sizeof(tExprNode));
if (pNode->nodeType == TEXPR_BINARYEXPR_NODE) {
tExprNode* pLeft = exprdup(pNode->_node.pLeft);
tExprNode* pRight = exprdup(pNode->_node.pRight);
pCloned->_node.pLeft = pLeft;
pCloned->_node.pRight = pRight;
pCloned->_node.optr = pNode->_node.optr;
} else if (pNode->nodeType == TEXPR_VALUE_NODE) {
pCloned->pVal = taosMemoryCalloc(1, sizeof(SVariant));
taosVariantAssign(pCloned->pVal, pNode->pVal);
} else if (pNode->nodeType == TEXPR_COL_NODE) {
pCloned->pSchema = taosMemoryCalloc(1, sizeof(SSchema));
*pCloned->pSchema = *pNode->pSchema;
} else if (pNode->nodeType == TEXPR_FUNCTION_NODE) {
strcpy(pCloned->_function.functionName, pNode->_function.functionName);
int32_t num = pNode->_function.num;
pCloned->_function.num = num;
pCloned->_function.pChild = taosMemoryCalloc(num, POINTER_BYTES);
for(int32_t i = 0; i < num; ++i) {
pCloned->_function.pChild[i] = exprdup(pNode->_function.pChild[i]);
}
}
pCloned->nodeType = pNode->nodeType;
return pCloned;
}
source/libs/nodes/src/nodesCodeFuncs.c
浏览文件 @ eb016e01
...@@ -1142,9 +1142,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { ...@@ -1142,9 +1142,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { return physiScanNodeToJson(pObj, pJson); } static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { return physiTableScanNodeToJson(pObj, pJson); }
static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiScanNode(pJson, pObj); } static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiTableScanNode(pJson, pObj); }
static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet"; static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet";
static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite"; static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite";
......
source/libs/parser/inc/parAst.h
浏览文件 @ eb016e01
...@@ -32,6 +32,7 @@ typedef struct SAstCreateContext { ...@@ -32,6 +32,7 @@ typedef struct SAstCreateContext {
bool notSupport; bool notSupport;
SNode* pRootNode; SNode* pRootNode;
int16_t placeholderNo; int16_t placeholderNo;
SArray* pPlaceholderValues;
int32_t errCode; int32_t errCode;
} SAstCreateContext; } SAstCreateContext;
......
source/libs/parser/src/parAstCreater.c
浏览文件 @ eb016e01
...@@ -44,6 +44,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { ...@@ -44,6 +44,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
pCxt->notSupport = false; pCxt->notSupport = false;
pCxt->pRootNode = NULL; pCxt->pRootNode = NULL;
pCxt->placeholderNo = 0; pCxt->placeholderNo = 0;
pCxt->pPlaceholderValues = NULL;
pCxt->errCode = TSDB_CODE_SUCCESS; pCxt->errCode = TSDB_CODE_SUCCESS;
} }
...@@ -299,6 +300,14 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera ...@@ -299,6 +300,14 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera
val->literal = strndup(pLiteral->z, pLiteral->n); val->literal = strndup(pLiteral->z, pLiteral->n);
CHECK_OUT_OF_MEM(val->literal); CHECK_OUT_OF_MEM(val->literal);
val->placeholderNo = ++pCxt->placeholderNo; val->placeholderNo = ++pCxt->placeholderNo;
if (NULL == pCxt->pPlaceholderValues) {
pCxt->pPlaceholderValues = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
if (NULL == pCxt->pPlaceholderValues) {
nodesDestroyNode(val);
return NULL;
}
}
taosArrayPush(pCxt->pPlaceholderValues, &val);
return (SNode*)val; return (SNode*)val;
} }
......
source/libs/parser/src/parAstParser.c
浏览文件 @ eb016e01
...@@ -81,6 +81,8 @@ abort_parse: ...@@ -81,6 +81,8 @@ abort_parse:
} }
(*pQuery)->pRoot = cxt.pRootNode; (*pQuery)->pRoot = cxt.pRootNode;
(*pQuery)->placeholderNum = cxt.placeholderNo; (*pQuery)->placeholderNum = cxt.placeholderNo;
TSWAP((*pQuery)->pPlaceholderValues, cxt.pPlaceholderValues);
} }
taosArrayDestroy(cxt.pPlaceholderValues);
return cxt.errCode; return cxt.errCode;
} }
source/libs/parser/src/parInsert.c
浏览文件 @ eb016e01
...@@ -1071,6 +1071,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { ...@@ -1071,6 +1071,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
int32_t tbNum = 0; int32_t tbNum = 0;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
bool autoCreateTbl = false; bool autoCreateTbl = false;
STableMeta *pMeta = NULL;
// for each table // for each table
while (1) { while (1) {
...@@ -1127,10 +1128,12 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { ...@@ -1127,10 +1128,12 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE,
sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta,
&dataBuf, NULL, &pCxt->createTblReq)); &dataBuf, NULL, &pCxt->createTblReq));
pMeta = pCxt->pTableMeta;
pCxt->pTableMeta = NULL;
if (TK_NK_LP == sToken.type) { if (TK_NK_LP == sToken.type) {
// pSql -> field1_name, ...) // pSql -> field1_name, ...)
CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pMeta)));
NEXT_TOKEN(pCxt->pSql, sToken); NEXT_TOKEN(pCxt->pSql, sToken);
} }
...@@ -1166,8 +1169,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { ...@@ -1166,8 +1169,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); memcpy(tags, &pCxt->tags, sizeof(pCxt->tags));
(*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj);
pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj);
memset(&pCxt->tags, 0, sizeof(pCxt->tags)); memset(&pCxt->tags, 0, sizeof(pCxt->tags));
pCxt->pVgroupsHashObj = NULL; pCxt->pVgroupsHashObj = NULL;
...@@ -1677,8 +1679,8 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD ...@@ -1677,8 +1679,8 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* colsSchema, SArray* cols, bool format, int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols, bool format,
STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen) { STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen) {
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle;
...@@ -1720,8 +1722,8 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* cols ...@@ -1720,8 +1722,8 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* cols
initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo);
int32_t rowNum = format ? taosArrayGetSize(colsFormat) : taosArrayGetSize(cols); int32_t rowNum = taosArrayGetSize(cols);
if (rowNum <= 0) { if(rowNum <= 0) {
return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); return buildInvalidOperationMsg(&pBuf, "cols size <= 0");
} }
ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum); ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum);
...@@ -1732,13 +1734,10 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* cols ...@@ -1732,13 +1734,10 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* cols
for (int32_t r = 0; r < rowNum; ++r) { for (int32_t r = 0; r < rowNum; ++r) {
STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header
tdSRowResetBuf(pBuilder, row); tdSRowResetBuf(pBuilder, row);
void* rowData = NULL; void *rowData = taosArrayGetP(cols, r);
size_t rowDataSize = 0; size_t rowDataSize = 0;
if (format) { if(format){
rowData = taosArrayGetP(colsFormat, r);
rowDataSize = taosArrayGetSize(rowData); rowDataSize = taosArrayGetSize(rowData);
} else {
rowData = taosArrayGetP(cols, r);
} }
// 1. set the parsed value from sql string // 1. set the parsed value from sql string
......
source/libs/parser/src/parser.c
浏览文件 @ eb016e01
...@@ -39,16 +39,99 @@ static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) { ...@@ -39,16 +39,99 @@ static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = authenticate(pCxt, *pQuery); code = authenticate(pCxt, *pQuery);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code && 0 == (*pQuery)->placeholderNum) {
code = translate(pCxt, *pQuery); code = translate(pCxt, *pQuery);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code && 0 == (*pQuery)->placeholderNum) {
code = calculateConstant(pCxt, *pQuery); code = calculateConstant(pCxt, *pQuery);
} }
return code; return code;
} }
int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery) { static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
if (pParam->is_null && 1 == *(pParam->is_null)) {
pVal->node.resType.type = TSDB_DATA_TYPE_NULL;
pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes;
return TSDB_CODE_SUCCESS;
}
int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes);
pVal->node.resType.type = pParam->buffer_type;
pVal->node.resType.bytes = inputSize;
switch (pParam->buffer_type) {
case TSDB_DATA_TYPE_BOOL:
pVal->datum.b = *((bool*)pParam->buffer);
break;
case TSDB_DATA_TYPE_TINYINT:
pVal->datum.i = *((int8_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_SMALLINT:
pVal->datum.i = *((int16_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_INT:
pVal->datum.i = *((int32_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_BIGINT:
pVal->datum.i = *((int64_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_FLOAT:
pVal->datum.d = *((float*)pParam->buffer);
break;
case TSDB_DATA_TYPE_DOUBLE:
pVal->datum.d = *((double*)pParam->buffer);
break;
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1);
if (NULL == pVal->datum.p) {
return TSDB_CODE_OUT_OF_MEMORY;
}
varDataSetLen(pVal->datum.p, pVal->node.resType.bytes);
strncpy(varDataVal(pVal->datum.p), (const char*)pParam->buffer, pVal->node.resType.bytes);
break;
case TSDB_DATA_TYPE_NCHAR: {
pVal->node.resType.bytes *= TSDB_NCHAR_SIZE;
pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1);
if (NULL == pVal->datum.p) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t output = 0;
if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes,
&output)) {
return errno;
}
varDataSetLen(pVal->datum.p, output);
pVal->node.resType.bytes = output;
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
pVal->datum.i = *((int64_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_UTINYINT:
pVal->datum.u = *((uint8_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_USMALLINT:
pVal->datum.u = *((uint16_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_UINT:
pVal->datum.u = *((uint32_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_UBIGINT:
pVal->datum.u = *((uint64_t*)pParam->buffer);
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
case TSDB_DATA_TYPE_MEDIUMBLOB:
// todo
default:
break;
}
pVal->translate = true;
return TSDB_CODE_SUCCESS;
}
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (isInsertSql(pCxt->pSql, pCxt->sqlLen)) { if (isInsertSql(pCxt->pSql, pCxt->sqlLen)) {
code = parseInsertSql(pCxt, pQuery); code = parseInsertSql(pCxt, pQuery);
...@@ -77,3 +160,29 @@ void qDestroyQuery(SQuery* pQueryNode) { ...@@ -77,3 +160,29 @@ void qDestroyQuery(SQuery* pQueryNode) {
int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) {
return extractResultSchema(pRoot, numOfCols, pSchema); return extractResultSchema(pRoot, numOfCols, pSchema);
} }
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) {
int32_t code = TSDB_CODE_SUCCESS;
if (colIdx < 0) {
int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues);
for (int32_t i = 0; i < size; ++i) {
code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
}
} else {
code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams);
}
return code;
}
int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery) {
int32_t code = translate(pCxt, pQuery);
if (TSDB_CODE_SUCCESS == code) {
code = calculateConstant(pCxt, pQuery);
}
return code;
}
source/libs/planner/src/planPhysiCreater.c
浏览文件 @ eb016e01
...@@ -460,9 +460,13 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp ...@@ -460,9 +460,13 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
memcpy(pTableScan->scanSeq, pScanLogicNode->scanSeq, sizeof(pScanLogicNode->scanSeq)); memcpy(pTableScan->scanSeq, pScanLogicNode->scanSeq, sizeof(pScanLogicNode->scanSeq));
pTableScan->scanRange = pScanLogicNode->scanRange; pTableScan->scanRange = pScanLogicNode->scanRange;
pTableScan->ratio = pScanLogicNode->ratio; pTableScan->ratio = pScanLogicNode->ratio;
if (pScanLogicNode->pVgroupList) {
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode);
pSubplan->execNodeStat.tableNum = pScanLogicNode->pVgroupList->vgroups[0].numOfTable; pSubplan->execNodeStat.tableNum = pScanLogicNode->pVgroupList->vgroups[0].numOfTable;
}
if (pCxt->pExecNodeList) {
taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode);
}
tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
pTableScan->dataRequired = pScanLogicNode->dataRequired; pTableScan->dataRequired = pScanLogicNode->dataRequired;
pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs); pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs);
...@@ -505,13 +509,12 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* ...@@ -505,13 +509,12 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
SPhysiNode** pPhyNode) { SPhysiNode** pPhyNode) {
SStreamScanPhysiNode* pScan = int32_t res = createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode);
(SStreamScanPhysiNode*)makePhysiNode(pCxt, pScanLogicNode->pMeta->tableInfo.precision, if (res == TSDB_CODE_SUCCESS) {
(SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); ENodeType type = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
if (NULL == pScan) { setNodeType(*pPhyNode, type);
return TSDB_CODE_OUT_OF_MEMORY;
} }
return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); return res;
} }
static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
...@@ -786,7 +789,7 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic ...@@ -786,7 +789,7 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic
} }
static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode,
SPhysiNode** pPhyNode) { SPhysiNode** pPhyNode) {
SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode( SScanPhysiNode* pScan = (SScanPhysiNode*)makePhysiNode(
pCxt, pExchangeLogicNode->precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); pCxt, pExchangeLogicNode->precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
if (NULL == pScan) { if (NULL == pScan) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
......
source/libs/planner/src/planner.c
浏览文件 @ eb016e01
...@@ -224,7 +224,7 @@ static int32_t calcConstList(SNodeList* pList) { ...@@ -224,7 +224,7 @@ static int32_t calcConstList(SNodeList* pList) {
} }
static bool isEmptyResultCond(SNode** pCond) { static bool isEmptyResultCond(SNode** pCond) {
if (QUERY_NODE_VALUE != nodeType(*pCond)) { if (NULL == *pCond || QUERY_NODE_VALUE != nodeType(*pCond)) {
return false; return false;
} }
if (((SValueNode*)*pCond)->datum.b) { if (((SValueNode*)*pCond)->datum.b) {
......
source/libs/planner/test/planStmtTest.cpp
浏览文件 @ eb016e01
...@@ -50,5 +50,5 @@ class PlanStmtTest : public PlannerTestBase { ...@@ -50,5 +50,5 @@ class PlanStmtTest : public PlannerTestBase {
TEST_F(PlanStmtTest, stmt) { TEST_F(PlanStmtTest, stmt) {
useDb("root", "test"); useDb("root", "test");
run("select * from t1 where c1 = ?"); // run("select * from t1 where c1 = ?");
} }
source/libs/planner/test/planTestUtil.cpp
浏览文件 @ eb016e01
...@@ -198,7 +198,7 @@ class PlannerTestBaseImpl { ...@@ -198,7 +198,7 @@ class PlannerTestBaseImpl {
cxt.pMsg = stmtEnv_.msgBuf_.data(); cxt.pMsg = stmtEnv_.msgBuf_.data();
cxt.msgLen = stmtEnv_.msgBuf_.max_size(); cxt.msgLen = stmtEnv_.msgBuf_.max_size();
DO_WITH_THROW(qParseQuerySql, &cxt, pQuery); DO_WITH_THROW(qParseSql, &cxt, pQuery);
res_.ast_ = toString((*pQuery)->pRoot); res_.ast_ = toString((*pQuery)->pRoot);
} }
......
source/libs/scalar/CMakeLists.txt
浏览文件 @ eb016e01
...@@ -8,7 +8,7 @@ target_include_directories( ...@@ -8,7 +8,7 @@ target_include_directories(
) )
target_link_libraries(scalar target_link_libraries(scalar
PRIVATE os util common nodes function qcom PRIVATE os util common nodes function qcom vnode
) )
if(${BUILD_TEST}) if(${BUILD_TEST})
......
source/libs/scalar/inc/sclInt.h
浏览文件 @ eb016e01
...@@ -26,6 +26,7 @@ typedef struct SScalarCtx { ...@@ -26,6 +26,7 @@ typedef struct SScalarCtx {
int32_t code; int32_t code;
SArray *pBlockList; /* element is SSDataBlock* */ SArray *pBlockList; /* element is SSDataBlock* */
SHashObj *pRes; /* element is SScalarParam */ SHashObj *pRes; /* element is SScalarParam */
void *param; // additional parameter (meta actually) for acquire value such as tbname/tags values
} SScalarCtx; } SScalarCtx;
...@@ -49,6 +50,7 @@ typedef struct SScalarCtx { ...@@ -49,6 +50,7 @@ typedef struct SScalarCtx {
int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
void sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode);
#define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) #define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type)
#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) #define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes)
......
source/libs/scalar/src/filter.c
浏览文件 @ eb016e01
...@@ -3505,19 +3505,6 @@ int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode* pNode) { ...@@ -3505,19 +3505,6 @@ int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode* pNode) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void fltConvertToTsValueNode(SFltTreeStat *stat, SValueNode* valueNode) {
char *timeStr = valueNode->datum.p;
if (convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, stat->precision, &valueNode->datum.i) !=
TSDB_CODE_SUCCESS) {
valueNode->datum.i = 0;
}
taosMemoryFree(timeStr);
valueNode->typeData = valueNode->datum.i;
valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
}
EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
SFltTreeStat *stat = (SFltTreeStat *)pContext; SFltTreeStat *stat = (SFltTreeStat *)pContext;
...@@ -3566,7 +3553,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { ...@@ -3566,7 +3553,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
fltConvertToTsValueNode(stat, valueNode); sclConvertToTsValueNode(stat->precision, valueNode);
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
...@@ -3614,6 +3601,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { ...@@ -3614,6 +3601,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
if (FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP) && node->opType >= OP_TYPE_NOT_EQUAL) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
if (NULL == node->pRight) { if (NULL == node->pRight) {
if (scalarGetOperatorParamNum(node->opType) > 1) { if (scalarGetOperatorParamNum(node->opType) > 1) {
fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType);
...@@ -3695,7 +3687,7 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { ...@@ -3695,7 +3687,7 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
for (int32_t i = 0; i < nodeNum; ++i) { for (int32_t i = 0; i < nodeNum; ++i) {
SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i); SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i);
fltConvertToTsValueNode(pStat, valueNode); sclConvertToTsValueNode(pStat->precision, valueNode);
} }
_return: _return:
......
source/libs/scalar/src/scalar.c
浏览文件 @ eb016e01
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include "tdatablock.h" #include "tdatablock.h"
#include "scalar.h" #include "scalar.h"
#include "tudf.h" #include "tudf.h"
#include "ttime.h"
int32_t scalarGetOperatorParamNum(EOperatorType type) { int32_t scalarGetOperatorParamNum(EOperatorType type) {
if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type
...@@ -19,6 +20,19 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) { ...@@ -19,6 +20,19 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) {
return 2; return 2;
} }
void sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) {
char *timeStr = valueNode->datum.p;
if (convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &valueNode->datum.i) !=
TSDB_CODE_SUCCESS) {
valueNode->datum.i = 0;
}
taosMemoryFree(timeStr);
valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
}
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
if (pColumnData == NULL) { if (pColumnData == NULL) {
...@@ -251,6 +265,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t ...@@ -251,6 +265,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
*rowNum = param->numOfRows; *rowNum = param->numOfRows;
} }
param->param = ctx->param;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -535,7 +550,7 @@ EDealRes sclRewriteBasedOnOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opT ...@@ -535,7 +550,7 @@ EDealRes sclRewriteBasedOnOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opT
} }
EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) {
SOperatorNode *node = (SOperatorNode *)*pNode; SOperatorNode *node = (SOperatorNode *)*pNode;
if (node->pLeft && (QUERY_NODE_VALUE == nodeType(node->pLeft))) { if (node->pLeft && (QUERY_NODE_VALUE == nodeType(node->pLeft))) {
...@@ -543,6 +558,11 @@ EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) { ...@@ -543,6 +558,11 @@ EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) {
if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) {
return sclRewriteBasedOnOptr(pNode, ctx, node->opType); return sclRewriteBasedOnOptr(pNode, ctx, node->opType);
} }
if (IS_STR_DATA_TYPE(valueNode->node.resType.type) && node->pRight && nodesIsExprNode(node->pRight)
&& ((SExprNode*)node->pRight)->resType.type == TSDB_DATA_TYPE_TIMESTAMP) {
sclConvertToTsValueNode(((SExprNode*)node->pRight)->resType.precision, valueNode);
}
} }
if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) { if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) {
...@@ -550,6 +570,11 @@ EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) { ...@@ -550,6 +570,11 @@ EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) {
if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) {
return sclRewriteBasedOnOptr(pNode, ctx, node->opType); return sclRewriteBasedOnOptr(pNode, ctx, node->opType);
} }
if (IS_STR_DATA_TYPE(valueNode->node.resType.type) && node->pLeft && nodesIsExprNode(node->pLeft)
&& ((SExprNode*)node->pLeft)->resType.type == TSDB_DATA_TYPE_TIMESTAMP) {
sclConvertToTsValueNode(((SExprNode*)node->pLeft)->resType.precision, valueNode);
}
} }
if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) { if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) {
...@@ -672,7 +697,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { ...@@ -672,7 +697,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
SOperatorNode *node = (SOperatorNode *)*pNode; SOperatorNode *node = (SOperatorNode *)*pNode;
if ((!SCL_IS_CONST_NODE(node->pLeft)) || (!SCL_IS_CONST_NODE(node->pRight))) { if ((!SCL_IS_CONST_NODE(node->pLeft)) || (!SCL_IS_CONST_NODE(node->pRight))) {
return sclRewriteOperatorForNullValue(pNode, ctx); return sclRewriteNonConstOperator(pNode, ctx);
} }
SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
...@@ -885,7 +910,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { ...@@ -885,7 +910,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
} }
int32_t code = 0; int32_t code = 0;
SScalarCtx ctx = {.code = 0, .pBlockList = pBlockList}; SScalarCtx ctx = {.code = 0, .pBlockList = pBlockList, .param = pDst->param};
// TODO: OPT performance // TODO: OPT performance
ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
......
source/libs/scalar/src/sclfunc.c
浏览文件 @ eb016e01
#include "function.h" #include "function.h"
#include "scalar.h" #include "scalar.h"
#include "tdatablock.h"
#include "ttime.h"
#include "sclInt.h" #include "sclInt.h"
#include "sclvector.h" #include "sclvector.h"
#include "tdatablock.h"
#include "tjson.h" #include "tjson.h"
#include "ttime.h"
#include "vnode.h"
typedef float (*_float_fn)(float); typedef float (*_float_fn)(float);
typedef double (*_double_fn)(double); typedef double (*_double_fn)(double);
...@@ -1512,6 +1513,21 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p ...@@ -1512,6 +1513,21 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
ASSERT(inputNum == 1); ASSERT(inputNum == 1);
colDataAppend(pOutput->columnData, pOutput->numOfRows, colDataGetData(pInput->columnData, 0), false);
SMetaReader mr = {0};
metaReaderInit(&mr, pInput->param, 0);
uint64_t uid = *(uint64_t *)colDataGetData(pInput->columnData, 0);
metaGetTableEntryByUid(&mr, uid);
char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(str, mr.me.name);
metaReaderClear(&mr);
for(int32_t i = 0; i < pInput->numOfRows; ++i) {
colDataAppend(pOutput->columnData, pOutput->numOfRows + i, str, false);
}
pOutput->numOfRows += pInput->numOfRows;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
source/libs/stream/src/tstream.c
浏览文件 @ eb016e01
...@@ -154,7 +154,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in ...@@ -154,7 +154,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
// sink // sink
if (pTask->sinkType == TASK_SINK__TABLE) { if (pTask->sinkType == TASK_SINK__TABLE) {
/*blockDebugShowData(pRes);*/ blockDebugShowData(pRes);
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes); pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes);
} else if (pTask->sinkType == TASK_SINK__SMA) { } else if (pTask->sinkType == TASK_SINK__SMA) {
pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes); pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes);
......
source/libs/wal/src/walRead.c
浏览文件 @ eb016e01
...@@ -54,8 +54,8 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { ...@@ -54,8 +54,8 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) {
return 0; return 0;
} }
static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { static int64_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
int ret = 0; int64_t ret = 0;
TdFilePtr pIdxTFile = pRead->pReadIdxTFile; TdFilePtr pIdxTFile = pRead->pReadIdxTFile;
TdFilePtr pLogTFile = pRead->pReadLogTFile; TdFilePtr pLogTFile = pRead->pReadLogTFile;
...@@ -68,14 +68,14 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i ...@@ -68,14 +68,14 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
wError("failed to seek idx file, ver %ld, pos: %ld, since %s", ver, offset, terrstr()); wError("failed to seek idx file, ver %ld, pos: %ld, since %s", ver, offset, terrstr());
return -1; return -1;
} }
SWalIdxEntry entry; SWalIdxEntry entry = {0};
if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) { if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) {
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("failed to read idx file, since %s", terrstr()); wError("failed to read idx file, since %s", terrstr());
} else { } else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
wError("read idx file incompletely, read bytes %d, bytes should be %lu", ret, sizeof(SWalIdxEntry)); wError("read idx file incompletely, read bytes %ld, bytes should be %lu", ret, sizeof(SWalIdxEntry));
} }
return -1; return -1;
} }
...@@ -156,7 +156,7 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { ...@@ -156,7 +156,7 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capacity = capacity; } void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capacity = capacity; }
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) { int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) {
int32_t code; int64_t code;
// TODO: valid ver // TODO: valid ver
if (ver > pRead->pWal->vers.commitVer) { if (ver > pRead->pWal->vers.commitVer) {
...@@ -187,7 +187,7 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) { ...@@ -187,7 +187,7 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) {
} }
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead) { int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead) {
int32_t code; int64_t code;
ASSERT(pRead->curVersion == pHead->head.version); ASSERT(pRead->curVersion == pHead->head.version);
...@@ -214,23 +214,24 @@ int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead) { ...@@ -214,23 +214,24 @@ int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead) {
return -1; return -1;
} }
*ppHead = ptr; *ppHead = ptr;
pReadHead = &((*ppHead)->head);
pRead->capacity = pReadHead->bodyLen; pRead->capacity = pReadHead->bodyLen;
} }
if (pReadHead->bodyLen != taosReadFile(pRead->pReadLogTFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen != taosReadFile(pRead->pReadLogTFile, pReadHead->body, pReadHead->bodyLen)) {
ASSERT(0);
return -1; return -1;
} }
if (pReadHead->version != ver) { if (pReadHead->version != ver) {
wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver);
ver);
pRead->curVersion = -1; pRead->curVersion = -1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; return -1;
} }
if (walValidBodyCksum(*ppHead) != 0) { if (walValidBodyCksum(*ppHead) != 0) {
wError("unexpected wal log version: % " PRId64 ", since body checksum not passed", ver); wError("wal fetch body error: % " PRId64 ", since body checksum not passed", ver);
pRead->curVersion = -1; pRead->curVersion = -1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; return -1;
...@@ -257,7 +258,7 @@ int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **p ...@@ -257,7 +258,7 @@ int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **p
} }
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
int code; int64_t code;
// TODO: check wal life // TODO: check wal life
if (pRead->curVersion != ver) { if (pRead->curVersion != ver) {
if (walReadSeekVer(pRead, ver) < 0) { if (walReadSeekVer(pRead, ver) < 0) {
......
source/libs/wal/src/walSeek.c
浏览文件 @ eb016e01
...@@ -19,8 +19,8 @@ ...@@ -19,8 +19,8 @@
#include "tref.h" #include "tref.h"
#include "walInt.h" #include "walInt.h"
static int walSeekWritePos(SWal* pWal, int64_t ver) { static int64_t walSeekWritePos(SWal* pWal, int64_t ver) {
int code = 0; int64_t code = 0;
TdFilePtr pIdxTFile = pWal->pWriteIdxTFile; TdFilePtr pIdxTFile = pWal->pWriteIdxTFile;
TdFilePtr pLogTFile = pWal->pWriteLogTFile; TdFilePtr pLogTFile = pWal->pWriteLogTFile;
...@@ -45,7 +45,7 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { ...@@ -45,7 +45,7 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
return code; return 0;
} }
int walSetWrite(SWal* pWal) { int walSetWrite(SWal* pWal) {
...@@ -124,7 +124,7 @@ int walChangeWrite(SWal* pWal, int64_t ver) { ...@@ -124,7 +124,7 @@ int walChangeWrite(SWal* pWal, int64_t ver) {
} }
int walSeekWriteVer(SWal* pWal, int64_t ver) { int walSeekWriteVer(SWal* pWal, int64_t ver) {
int code; int64_t code;
if (ver == pWal->vers.lastVer) { if (ver == pWal->vers.lastVer) {
return 0; return 0;
} }
......
source/libs/wal/src/walWrite.c
浏览文件 @ eb016e01
...@@ -30,7 +30,7 @@ int32_t walCommit(SWal *pWal, int64_t ver) { ...@@ -30,7 +30,7 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
} }
int32_t walRollback(SWal *pWal, int64_t ver) { int32_t walRollback(SWal *pWal, int64_t ver) {
int code; int64_t code;
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
terrno = TSDB_CODE_WAL_INVALID_VER; terrno = TSDB_CODE_WAL_INVALID_VER;
......
source/os/src/osFile.c
浏览文件 @ eb016e01
...@@ -246,11 +246,11 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { ...@@ -246,11 +246,11 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0;
access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0;
access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0;
#ifdef WINDOWS #ifdef WINDOWS
fd = _open(path, access, _S_IREAD|_S_IWRITE); fd = _open(path, access, _S_IREAD | _S_IWRITE);
#else #else
fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO);
#endif #endif
if (fd == -1) { if (fd == -1) {
return NULL; return NULL;
} }
...@@ -310,9 +310,6 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { ...@@ -310,9 +310,6 @@ int64_t taosCloseFile(TdFilePtr *ppFile) {
} }
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
if (pFile == NULL) {
return 0;
}
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockRdlock(&(pFile->rwlock)); taosThreadRwlockRdlock(&(pFile->rwlock));
#endif #endif
...@@ -359,7 +356,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) ...@@ -359,7 +356,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
assert(pFile->fd >= 0); // Please check if you have closed the file. assert(pFile->fd >= 0); // Please check if you have closed the file.
#ifdef WINDOWS #ifdef WINDOWS
size_t pos = lseek(pFile->fd, 0, SEEK_CUR); size_t pos = lseek(pFile->fd, 0, SEEK_CUR);
lseek(pFile->fd, (long)offset, SEEK_SET); lseek(pFile->fd, offset, SEEK_SET);
int64_t ret = read(pFile->fd, buf, count); int64_t ret = read(pFile->fd, buf, count);
lseek(pFile->fd, pos, SEEK_SET); lseek(pFile->fd, pos, SEEK_SET);
#else #else
...@@ -372,9 +369,6 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) ...@@ -372,9 +369,6 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
} }
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
if (pFile == NULL) {
return 0;
}
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockWrlock(&(pFile->rwlock)); taosThreadRwlockWrlock(&(pFile->rwlock));
#endif #endif
...@@ -406,14 +400,11 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { ...@@ -406,14 +400,11 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
} }
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
if (pFile == NULL) {
return 0;
}
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockRdlock(&(pFile->rwlock)); taosThreadRwlockRdlock(&(pFile->rwlock));
#endif #endif
assert(pFile->fd >= 0); // Please check if you have closed the file. assert(pFile->fd >= 0); // Please check if you have closed the file.
int64_t ret = lseek(pFile->fd, (long)offset, whence); int64_t ret = lseek(pFile->fd, offset, whence);
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock)); taosThreadRwlockUnlock(&(pFile->rwlock));
#endif #endif
...@@ -424,9 +415,6 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { ...@@ -424,9 +415,6 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
#ifdef WINDOWS #ifdef WINDOWS
return 0; return 0;
#else #else
if (pFile == NULL) {
return 0;
}
assert(pFile->fd >= 0); // Please check if you have closed the file. assert(pFile->fd >= 0); // Please check if you have closed the file.
struct stat fileStat; struct stat fileStat;
...@@ -451,9 +439,6 @@ int32_t taosLockFile(TdFilePtr pFile) { ...@@ -451,9 +439,6 @@ int32_t taosLockFile(TdFilePtr pFile) {
#ifdef WINDOWS #ifdef WINDOWS
return 0; return 0;
#else #else
if (pFile == NULL) {
return 0;
}
assert(pFile->fd >= 0); // Please check if you have closed the file. assert(pFile->fd >= 0); // Please check if you have closed the file.
return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB);
...@@ -464,9 +449,6 @@ int32_t taosUnLockFile(TdFilePtr pFile) { ...@@ -464,9 +449,6 @@ int32_t taosUnLockFile(TdFilePtr pFile) {
#ifdef WINDOWS #ifdef WINDOWS
return 0; return 0;
#else #else
if (pFile == NULL) {
return 0;
}
assert(pFile->fd >= 0); // Please check if you have closed the file. assert(pFile->fd >= 0); // Please check if you have closed the file.
return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB);
...@@ -667,7 +649,7 @@ int32_t taosUmaskFile(int32_t maskVal) { ...@@ -667,7 +649,7 @@ int32_t taosUmaskFile(int32_t maskVal) {
int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; }
int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
if (pFile == NULL || ptrBuf == NULL ) { if (pFile == NULL || ptrBuf == NULL) {
return -1; return -1;
} }
if (*ptrBuf != NULL) { if (*ptrBuf != NULL) {
...@@ -689,7 +671,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { ...@@ -689,7 +671,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
#endif #endif
} }
int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) {
if (pFile == NULL || buf == NULL ) { if (pFile == NULL || buf == NULL) {
return -1; return -1;
} }
assert(pFile->fp != NULL); assert(pFile->fp != NULL);
......
tests/script/api/batchprepare.c
浏览文件 @ eb016e01
...@@ -20,6 +20,11 @@ typedef struct { ...@@ -20,6 +20,11 @@ typedef struct {
bool enclose; bool enclose;
} OperInfo; } OperInfo;
typedef struct {
char* funcName;
int32_t paramNum;
} FuncInfo;
typedef enum { typedef enum {
BP_BIND_TAG = 1, BP_BIND_TAG = 1,
BP_BIND_COL, BP_BIND_COL,
...@@ -44,6 +49,13 @@ OperInfo operInfo[] = { ...@@ -44,6 +49,13 @@ OperInfo operInfo[] = {
int32_t operatorList[] = {0, 1, 2, 3, 4, 5, 6, 7}; int32_t operatorList[] = {0, 1, 2, 3, 4, 5, 6, 7};
int32_t varoperatorList[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; int32_t varoperatorList[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
FuncInfo funcInfo[] = {
{"count", 1},
{"sum", 1},
{"min", 1},
{"sin", 1},
};
char *bpStbPrefix = "st"; char *bpStbPrefix = "st";
char *bpTbPrefix = "t"; char *bpTbPrefix = "t";
int32_t bpDefaultStbId = 1; int32_t bpDefaultStbId = 1;
...@@ -154,7 +166,7 @@ CaseCfg gCase[] = { ...@@ -154,7 +166,7 @@ CaseCfg gCase[] = {
{"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1},
{"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2},
{"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 2, 10, 1, 3, 0, 0, 1, 2},
}; };
...@@ -179,6 +191,8 @@ typedef struct { ...@@ -179,6 +191,8 @@ typedef struct {
int32_t* bindTagTypeList; int32_t* bindTagTypeList;
int32_t optrIdxListNum; int32_t optrIdxListNum;
int32_t* optrIdxList; int32_t* optrIdxList;
int32_t funcIdxListNum;
int32_t* funcIdxList;
int32_t runTimes; int32_t runTimes;
int32_t caseIdx; // static case idx int32_t caseIdx; // static case idx
int32_t caseNum; // num in static case list int32_t caseNum; // num in static case list
...@@ -186,7 +200,7 @@ typedef struct { ...@@ -186,7 +200,7 @@ typedef struct {
int32_t caseRunNum; // total run case num int32_t caseRunNum; // total run case num
} CaseCtrl; } CaseCtrl;
#if 1 #if 0
CaseCtrl gCaseCtrl = { // default CaseCtrl gCaseCtrl = { // default
.bindNullNum = 0, .bindNullNum = 0,
.printCreateTblSql = false, .printCreateTblSql = false,
...@@ -203,6 +217,8 @@ CaseCtrl gCaseCtrl = { // default ...@@ -203,6 +217,8 @@ CaseCtrl gCaseCtrl = { // default
.bindTagTypeList = NULL, .bindTagTypeList = NULL,
.optrIdxListNum = 0, .optrIdxListNum = 0,
.optrIdxList = NULL, .optrIdxList = NULL,
.funcIdxListNum = 0,
.funcIdxList = NULL,
.checkParamNum = false, .checkParamNum = false,
.printRes = false, .printRes = false,
.runTimes = 0, .runTimes = 0,
...@@ -241,7 +257,7 @@ CaseCtrl gCaseCtrl = { ...@@ -241,7 +257,7 @@ CaseCtrl gCaseCtrl = {
}; };
#endif #endif
#if 0 #if 1
CaseCtrl gCaseCtrl = { // query case with specified col&oper CaseCtrl gCaseCtrl = { // query case with specified col&oper
.bindNullNum = 0, .bindNullNum = 0,
.printCreateTblSql = false, .printCreateTblSql = false,
...@@ -255,14 +271,14 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper ...@@ -255,14 +271,14 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper
.optrIdxListNum = 0, .optrIdxListNum = 0,
.optrIdxList = NULL, .optrIdxList = NULL,
.checkParamNum = false, .checkParamNum = false,
.printRes = false, .printRes = true,
.runTimes = 0, .runTimes = 0,
.caseRunIdx = -1, .caseRunIdx = -1,
.optrIdxListNum = 0, .optrIdxListNum = 0,
.optrIdxList = NULL, .optrIdxList = NULL,
.bindColTypeNum = 0, .bindColTypeNum = 0,
.bindColTypeList = NULL, .bindColTypeList = NULL,
.caseIdx = 23, .caseIdx = 24,
.caseNum = 1, .caseNum = 1,
.caseRunNum = 1, .caseRunNum = 1,
}; };
...@@ -513,66 +529,92 @@ void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType, int32 ...@@ -513,66 +529,92 @@ void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType, int32
} }
break; break;
default: default:
printf("!!!invalid paramNum:%d\n", pInfo->paramNum); printf("!!!invalid operator paramNum:%d\n", pInfo->paramNum);
exit(1); exit(1);
} }
} }
void generateQueryCondSQL(BindData *data, int32_t tblIdx) { void bpAppendFunctionParam(BindData *data, int32_t *len, int32_t dataType, int32_t idx) {
int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx); FuncInfo *pInfo = NULL;
if (!gCurCase->fullCol) {
for (int c = 0; c < gCurCase->bindColNum; ++c) { if (gCaseCtrl.funcIdxListNum > 0) {
if (c) { pInfo = &funcInfo[gCaseCtrl.funcIdxList[idx]];
len += sprintf(data->sql + len, " and "); } else {
pInfo = &funcInfo[rand() % tListLen(funcInfo)];
} }
switch (data->pBind[c].buffer_type) {
switch (pInfo->paramNum) {
case 1:
*len += sprintf(data->sql + *len, " %s(?)", pInfo->funcName);
break;
default:
printf("!!!invalid function paramNum:%d\n", pInfo->paramNum);
exit(1);
}
}
int32_t bpAppendColumnName(BindData *data, int32_t type, int32_t len) {
switch (type) {
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
len += sprintf(data->sql + len, "booldata"); return sprintf(data->sql + len, "booldata");
break; break;
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
len += sprintf(data->sql + len, "tinydata"); return sprintf(data->sql + len, "tinydata");
break; break;
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
len += sprintf(data->sql + len, "smalldata"); return sprintf(data->sql + len, "smalldata");
break; break;
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
len += sprintf(data->sql + len, "intdata"); return sprintf(data->sql + len, "intdata");
break; break;
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
len += sprintf(data->sql + len, "bigdata"); return sprintf(data->sql + len, "bigdata");
break; break;
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
len += sprintf(data->sql + len, "floatdata"); return sprintf(data->sql + len, "floatdata");
break; break;
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
len += sprintf(data->sql + len, "doubledata"); return sprintf(data->sql + len, "doubledata");
break; break;
case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARCHAR:
len += sprintf(data->sql + len, "binarydata"); return sprintf(data->sql + len, "binarydata");
break; break;
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
len += sprintf(data->sql + len, "ts"); return sprintf(data->sql + len, "ts");
break; break;
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
len += sprintf(data->sql + len, "nchardata"); return sprintf(data->sql + len, "nchardata");
break; break;
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
len += sprintf(data->sql + len, "utinydata"); return sprintf(data->sql + len, "utinydata");
break; break;
case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_USMALLINT:
len += sprintf(data->sql + len, "usmalldata"); return sprintf(data->sql + len, "usmalldata");
break; break;
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
len += sprintf(data->sql + len, "uintdata"); return sprintf(data->sql + len, "uintdata");
break; break;
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
len += sprintf(data->sql + len, "ubigdata"); return sprintf(data->sql + len, "ubigdata");
break; break;
default: default:
printf("!!!invalid col type:%d", data->pBind[c].buffer_type); printf("!!!invalid col type:%d", type);
exit(1); exit(1);
} }
return 0;
}
void generateQueryCondSQL(BindData *data, int32_t tblIdx) {
int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx);
if (!gCurCase->fullCol) {
for (int c = 0; c < gCurCase->bindColNum; ++c) {
if (c) {
len += sprintf(data->sql + len, " and ");
}
len += bpAppendColumnName(data, data->pBind[c].buffer_type, len);
bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c);
} }
} }
...@@ -582,64 +624,50 @@ void generateQueryCondSQL(BindData *data, int32_t tblIdx) { ...@@ -582,64 +624,50 @@ void generateQueryCondSQL(BindData *data, int32_t tblIdx) {
} }
} }
void bpGenerateConstInOpSQL(BindData *data, int32_t tblIdx) {
int32_t len = 0;
len = sprintf(data->sql, "select ");
void generateQueryMiscSQL(BindData *data, int32_t tblIdx) {
int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx);
if (!gCurCase->fullCol) {
for (int c = 0; c < gCurCase->bindColNum; ++c) { for (int c = 0; c < gCurCase->bindColNum; ++c) {
if (c) { if (c) {
len += sprintf(data->sql + len, " and "); len += sprintf(data->sql + len, ", ");
}
switch (data->pBind[c].buffer_type) {
case TSDB_DATA_TYPE_BOOL:
len += sprintf(data->sql + len, "booldata");
break;
case TSDB_DATA_TYPE_TINYINT:
len += sprintf(data->sql + len, "tinydata");
break;
case TSDB_DATA_TYPE_SMALLINT:
len += sprintf(data->sql + len, "smalldata");
break;
case TSDB_DATA_TYPE_INT:
len += sprintf(data->sql + len, "intdata");
break;
case TSDB_DATA_TYPE_BIGINT:
len += sprintf(data->sql + len, "bigdata");
break;
case TSDB_DATA_TYPE_FLOAT:
len += sprintf(data->sql + len, "floatdata");
break;
case TSDB_DATA_TYPE_DOUBLE:
len += sprintf(data->sql + len, "doubledata");
break;
case TSDB_DATA_TYPE_VARCHAR:
len += sprintf(data->sql + len, "binarydata");
break;
case TSDB_DATA_TYPE_TIMESTAMP:
len += sprintf(data->sql + len, "ts");
break;
case TSDB_DATA_TYPE_NCHAR:
len += sprintf(data->sql + len, "nchardata");
break;
case TSDB_DATA_TYPE_UTINYINT:
len += sprintf(data->sql + len, "utinydata");
break;
case TSDB_DATA_TYPE_USMALLINT:
len += sprintf(data->sql + len, "usmalldata");
break;
case TSDB_DATA_TYPE_UINT:
len += sprintf(data->sql + len, "uintdata");
break;
case TSDB_DATA_TYPE_UBIGINT:
len += sprintf(data->sql + len, "ubigdata");
break;
default:
printf("!!!invalid col type:%d", data->pBind[c].buffer_type);
exit(1);
} }
len += bpAppendColumnName(data, data->pBind[c].buffer_type, len);
bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c);
} }
len += sprintf(data->sql + len, " from %s%d", bpTbPrefix, tblIdx);
}
void bpGenerateConstInFuncSQL(BindData *data, int32_t tblIdx) {
int32_t len = 0;
len = sprintf(data->sql, "select ");
for (int c = 0; c < gCurCase->bindColNum; ++c) {
if (c) {
len += sprintf(data->sql + len, ", ");
}
bpAppendFunctionParam(data, &len, data->pBind[c].buffer_type, c);
}
len += sprintf(data->sql + len, " from %s%d", bpTbPrefix, tblIdx);
}
void generateQueryMiscSQL(BindData *data, int32_t tblIdx) {
switch(tblIdx) {
case 0:
bpGenerateConstInOpSQL(data, tblIdx);
break;
case 1:
//TODO FILL TEST
default:
bpGenerateConstInFuncSQL(data, tblIdx);
break;
} }
if (gCaseCtrl.printStmtSql) { if (gCaseCtrl.printStmtSql) {
......
tests/system-test/0-others/user_control.py 0 → 100644
浏览文件 @ eb016e01
import taos
import sys
import inspect
import traceback
from util.log import *
from util.sql import *
from util.cases import *
PRIVILEGES_ALL = "ALL"
PRIVILEGES_READ = "READ"
PRIVILEGES_WRITE = "WRITE"
class TDconnect:
def __init__(self,
host = None,
port = None,
user = None,
password = None,
database = None,
config = None,
) -> None:
self._conn = None
self._host = host
self._user = user
self._password = password
self._database = database
self._port = port
self._config = config
def __enter__(self):
self._conn = taos.connect(
host =self._host,
port =self._port,
user =self._user,
password=self._password,
database=self._database,
config =self._config
)
self.cursor = self._conn.cursor()
return self
def error(self, sql):
expectErrNotOccured = True
try:
self.cursor.execute(sql)
except BaseException:
expectErrNotOccured = False
if expectErrNotOccured:
caller = inspect.getframeinfo(inspect.stack()[1][0])
tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, expect error not occured" )
else:
self.queryRows = 0
self.queryCols = 0
self.queryResult = None
tdLog.info(f"sql:{sql}, expect error occured")
def query(self, sql, row_tag=None):
# sourcery skip: raise-from-previous-error, raise-specific-error
self.sql = sql
try:
self.cursor.execute(sql)
self.queryResult = self.cursor.fetchall()
self.queryRows = len(self.queryResult)
self.queryCols = len(self.cursor.description)
except Exception as e:
caller = inspect.getframeinfo(inspect.stack()[1][0])
tdLog.notice(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, {repr(e)}")
traceback.print_exc()
raise Exception(repr(e))
if row_tag:
return self.queryResult
return self.queryRows
def __exit__(self, types, values, trace):
if self._conn:
self.cursor.close()
self._conn.close()
def taos_connect(
host = "127.0.0.1",
port = 6030,
user = "root",
passwd = "taosdata",
database= None,
config = None
):
return TDconnect(
host = host,
port=port,
user=user,
password=passwd,
database=database,
config=config
)
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
@property
def __user_list(self):
return [f"user_test{i}" for i in range(self.users_count) ]
@property
def __passwd_list(self):
return [f"taosdata{i}" for i in range(self.users_count) ]
@property
def __privilege(self):
return [ PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE ]
def __priv_level(self, dbname=None):
return f"{dbname}.*" if dbname else "*.*"
def create_user_current(self):
users = self.__user_list
passwds = self.__passwd_list
for i in range(self.users_count):
tdSql.execute(f"create user {users[i]} pass '{passwds[i]}' ")
tdSql.query("show users")
tdSql.checkRows(self.users_count + 1)
def create_user_err(self):
sqls = [
"create users u1 pass 'u1passwd' ",
"create user '' pass 'u1passwd' ",
"create user pass 'u1passwd' ",
"create user u1 pass u1passwd ",
"create user u1 password 'u1passwd' ",
"create user u1 pass u1passwd ",
"create user u1 pass '' ",
"create user u1 pass ' ' ",
"create user u1 pass ",
"create user u1 u2 pass 'u1passwd' 'u2passwd' ",
"create user u1 u2 pass 'u1passwd', 'u2passwd' ",
"create user u1, u2 pass 'u1passwd', 'u2passwd' ",
"create user u1, u2 pass 'u1passwd' 'u2passwd' ",
# length of user_name must <= 23
"create user u12345678901234567890123 pass 'u1passwd' " ,
# length of passwd must <= 128
"create user u1 pass 'u12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678' " ,
# password must have not " ' ~ ` \
"create user u1 pass 'u1passwd\\' " ,
"create user u1 pass 'u1passwd~' " ,
"create user u1 pass 'u1passwd\"' " ,
"create user u1 pass 'u1passwd\'' " ,
"create user u1 pass 'u1passwd`' " ,
# must after create a user named u1
"create user u1 pass 'u1passwd' " ,
]
tdSql.execute("create user u1 pass 'u1passwd' ")
for sql in sqls:
tdSql.error(sql)
def __alter_pass_sql(self, user, passwd):
return f'''ALTER USER {user} PASS '{passwd}' '''
def alter_pass_current(self):
self.__init_pass = True
for count, i in enumerate(range(self.users_count)):
if self.__init_pass:
tdSql.query(self.__alter_pass_sql(self.__user_list[i], f"new{self.__passwd_list[i]}"))
self.__init_pass = count != self.users_count - 1
else:
tdSql.query(self.__alter_pass_sql(self.__user_list[i], self.__passwd_list[i] ) )
self.__init_pass = count == self.users_count - 1
def alter_pass_err(self): # sourcery skip: remove-redundant-fstring
sqls = [
f"alter users {self.__user_list[0]} pass 'newpass' " ,
f"alter user {self.__user_list[0]} pass '' " ,
f"alter user {self.__user_list[0]} pass ' ' " ,
f"alter user anyuser pass 'newpass' " ,
f"alter user {self.__user_list[0]} pass " ,
f"alter user {self.__user_list[0]} password 'newpass' " ,
]
for sql in sqls:
tdSql.error(sql)
def grant_user_privileges(self, privilege, dbname=None, user_name="root"):
return f"GRANT {privilege} ON {self.__priv_level(dbname)} TO {user_name} "
def test_user_create(self):
self.create_user_current()
self.create_user_err()
def test_alter_pass(self):
self.alter_pass_current()
self.alter_pass_err()
def user_login(self, user, passwd):
login_except = False
try:
with taos_connect(user=user, passwd=passwd) as conn:
cursor = conn.cursor
except BaseException:
login_except = True
cursor = None
return login_except, cursor
def login_currrent(self, user, passwd):
login_except, _ = self.user_login(user, passwd)
if login_except:
tdLog.exit(f"connect failed, user: {user} and pass: {passwd} do not match!")
else:
tdLog.info("connect successfully, user and pass matched!")
def login_err(self, user, passwd):
login_except, _ = self.user_login(user, passwd)
if login_except:
tdLog.info("connect failed, except error occured!")
else:
tdLog.exit("connect successfully, except error not occrued!")
def __drop_user(self, user):
return f"DROP USER {user}"
def drop_user_current(self):
for user in self.__user_list:
tdSql.query(self.__drop_user(user))
def drop_user_error(self):
sqls = [
f"DROP {self.__user_list[0]}",
f"DROP user {self.__user_list[0]} {self.__user_list[1]}",
f"DROP user {self.__user_list[0]} , {self.__user_list[1]}",
f"DROP users {self.__user_list[0]} {self.__user_list[1]}",
f"DROP users {self.__user_list[0]} , {self.__user_list[1]}",
"DROP user root",
"DROP user abcde",
"DROP user ALL",
]
for sql in sqls:
tdSql.error(sql)
def test_drop_user(self):
# must drop err first
self.drop_user_error()
self.drop_user_current()
def run(self):
# 默认只有 root 用户
tdLog.printNoPrefix("==========step0: init, user list only has root account")
tdSql.query("show users")
tdSql.checkData(0, 0, "root")
tdSql.checkData(0, 1, "super")
# root用户权限
# 创建用户测试
tdLog.printNoPrefix("==========step1: create user test")
self.users_count = 5
self.test_user_create()
# 查看用户
tdLog.printNoPrefix("==========step2: show user test")
tdSql.query("show users")
tdSql.checkRows(self.users_count + 2)
# 密码登录认证
self.login_currrent(self.__user_list[0], self.__passwd_list[0])
self.login_err(self.__user_list[0], f"new{self.__passwd_list[0]}")
# 修改密码
tdLog.printNoPrefix("==========step3: alter user pass test")
self.test_alter_pass()
# 密码修改后的登录认证
tdLog.printNoPrefix("==========step4: check login test")
self.login_err(self.__user_list[0], self.__passwd_list[0])
self.login_currrent(self.__user_list[0], f"new{self.__passwd_list[0]}")
# 普通用户权限
# 密码登录
_, user = self.user_login(self.__user_list[0], f"new{self.__passwd_list[0]}")
with taos_connect(user=self.__user_list[0], passwd=f"new{self.__passwd_list[0]}") as conn:
user = conn
# 不能创建用户
tdLog.printNoPrefix("==========step5: normal user can not create user")
user.error("create use utest1 pass 'utest1pass'")
# 可以查看用户
tdLog.printNoPrefix("==========step6: normal user can show user")
user.query("show users")
assert user.queryRows == self.users_count + 2
# 不可以修改其他用户的密码
tdLog.printNoPrefix("==========step7: normal user can not alter other user pass")
user.error(self.__alter_pass_sql(self.__user_list[1], self.__passwd_list[1] ))
user.error("root", "taosdata_root")
# 可以修改自己的密码
tdLog.printNoPrefix("==========step8: normal user can alter owner pass")
user.query(self.__alter_pass_sql(self.__user_list[0], self.__passwd_list[0]))
# 不可以删除用户，包括自己
tdLog.printNoPrefix("==========step9: normal user can not drop any user ")
user.error(f"drop user {self.__user_list[0]}")
user.error(f"drop user {self.__user_list[1]}")
user.error("drop user root")
# root删除用户测试
tdLog.printNoPrefix("==========step10: super user drop normal user")
self.test_drop_user()
tdSql.query("show users")
tdSql.checkRows(1)
tdSql.checkData(0, 0, "root")
tdSql.checkData(0, 1, "super")
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())
tests/system-test/2-query/between.py
浏览文件 @ eb016e01
...@@ -45,16 +45,16 @@ class TDTestCase: ...@@ -45,16 +45,16 @@ class TDTestCase:
tdLog.printNoPrefix("==========step3:query timestamp type") tdLog.printNoPrefix("==========step3:query timestamp type")
# tdSql.query("select * from t1 where ts between now()-1m and now()+10m") tdSql.query("select * from t1 where ts between now()-1m and now()+10m")
# tdSql.checkRows(10) tdSql.checkRows(10)
# tdSql.query("select * from t1 where ts between '2021-01-01 00:00:00.000' and '2121-01-01 00:00:00.000'") tdSql.query("select * from t1 where ts between '2021-01-01 00:00:00.000' and '2121-01-01 00:00:00.000'")
# tdSql.checkRows(11) # tdSql.checkRows(11)
# tdSql.query("select * from t1 where ts between '1969-01-01 00:00:00.000' and '1969-12-31 23:59:59.999'") tdSql.query("select * from t1 where ts between '1969-01-01 00:00:00.000' and '1969-12-31 23:59:59.999'")
# tdSql.checkRows(0)
# tdSql.query("select * from t1 where ts between -2793600 and 31507199")
# tdSql.checkRows(0) # tdSql.checkRows(0)
# tdSql.query("select * from t1 where ts between 1609430400000 and 4765104000000") tdSql.query("select * from t1 where ts between -2793600 and 31507199")
# tdSql.checkRows(11) tdSql.checkRows(0)
tdSql.query("select * from t1 where ts between 1609430400000 and 4765104000000")
tdSql.checkRows(11)
tdLog.printNoPrefix("==========step4:query int type") tdLog.printNoPrefix("==========step4:query int type")
...@@ -68,11 +68,11 @@ class TDTestCase: ...@@ -68,11 +68,11 @@ class TDTestCase:
tdSql.checkRows(0) tdSql.checkRows(0)
# tdSql.query("select * from t1 where c1 between 0x64 and 0x69") # tdSql.query("select * from t1 where c1 between 0x64 and 0x69")
# tdSql.checkRows(6) # tdSql.checkRows(6)
# tdSql.query("select * from t1 where c1 not between 100 and 106") tdSql.query("select * from t1 where c1 not between 100 and 106")
# tdSql.checkRows(11) tdSql.checkRows(11)
tdSql.query(f"select * from t1 where c1 between {2**31-2} and {2**31+1}") tdSql.query(f"select * from t1 where c1 between {2**31-2} and {2**31+1}")
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.error(f"select * from t2 where c1 between null and {1-2**31}") tdSql.query(f"select * from t2 where c1 between null and {1-2**31}")
# tdSql.checkRows(3) # tdSql.checkRows(3)
tdSql.query(f"select * from t2 where c1 between {-2**31} and {1-2**31}") tdSql.query(f"select * from t2 where c1 between {-2**31} and {1-2**31}")
tdSql.checkRows(1) tdSql.checkRows(1)
...@@ -88,12 +88,12 @@ class TDTestCase: ...@@ -88,12 +88,12 @@ class TDTestCase:
tdSql.query("select * from t1 where c2 between 'DC3' and 'SYN'") tdSql.query("select * from t1 where c2 between 'DC3' and 'SYN'")
tdSql.checkRows(0) tdSql.checkRows(0)
tdSql.query("select * from t1 where c2 not between 0.1 and 0.2") tdSql.query("select * from t1 where c2 not between 0.1 and 0.2")
# tdSql.checkRows(11) tdSql.checkRows(11)
tdSql.query(f"select * from t1 where c2 between {pow(10,38)*3.4} and {pow(10,38)*3.4+1}") tdSql.query(f"select * from t1 where c2 between {pow(10,38)*3.4} and {pow(10,38)*3.4+1}")
# tdSql.checkRows(1) # tdSql.checkRows(1)
tdSql.query(f"select * from t2 where c2 between {-3.4*10**38-1} and {-3.4*10**38}") tdSql.query(f"select * from t2 where c2 between {-3.4*10**38-1} and {-3.4*10**38}")
# tdSql.checkRows(2) # tdSql.checkRows(2)
tdSql.error(f"select * from t2 where c2 between null and {-3.4*10**38}") tdSql.query(f"select * from t2 where c2 between null and {-3.4*10**38}")
# tdSql.checkRows(3) # tdSql.checkRows(3)
tdLog.printNoPrefix("==========step6:query bigint type") tdLog.printNoPrefix("==========step6:query bigint type")
...@@ -101,7 +101,7 @@ class TDTestCase: ...@@ -101,7 +101,7 @@ class TDTestCase:
tdSql.query(f"select * from t1 where c3 between {2**31} and {2**31+10}") tdSql.query(f"select * from t1 where c3 between {2**31} and {2**31+10}")
tdSql.checkRows(10) tdSql.checkRows(10)
tdSql.query(f"select * from t1 where c3 between {-2**63} and {2**63}") tdSql.query(f"select * from t1 where c3 between {-2**63} and {2**63}")
# tdSql.checkRows(11) tdSql.checkRows(11)
tdSql.query(f"select * from t1 where c3 between {2**31+10} and {2**31}") tdSql.query(f"select * from t1 where c3 between {2**31+10} and {2**31}")
tdSql.checkRows(0) tdSql.checkRows(0)
tdSql.query("select * from t1 where c3 between 'a' and 'z'") tdSql.query("select * from t1 where c3 between 'a' and 'z'")
...@@ -112,7 +112,7 @@ class TDTestCase: ...@@ -112,7 +112,7 @@ class TDTestCase:
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.query(f"select * from t2 where c3 between {-2**63} and {1-2**63}") tdSql.query(f"select * from t2 where c3 between {-2**63} and {1-2**63}")
# tdSql.checkRows(3) # tdSql.checkRows(3)
tdSql.error(f"select * from t2 where c3 between null and {1-2**63}") tdSql.query(f"select * from t2 where c3 between null and {1-2**63}")
# tdSql.checkRows(2) # tdSql.checkRows(2)
tdLog.printNoPrefix("==========step7:query double type") tdLog.printNoPrefix("==========step7:query double type")
...@@ -129,10 +129,10 @@ class TDTestCase: ...@@ -129,10 +129,10 @@ class TDTestCase:
tdSql.query("select * from t1 where c4 not between 1 and 2") tdSql.query("select * from t1 where c4 not between 1 and 2")
# tdSql.checkRows(0) # tdSql.checkRows(0)
tdSql.query(f"select * from t1 where c4 between {1.7*10**308} and {1.7*10**308+1}") tdSql.query(f"select * from t1 where c4 between {1.7*10**308} and {1.7*10**308+1}")
# tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.query(f"select * from t2 where c4 between {-1.7*10**308-1} and {-1.7*10**308}") tdSql.query(f"select * from t2 where c4 between {-1.7*10**308-1} and {-1.7*10**308}")
# tdSql.checkRows(3) # tdSql.checkRows(3)
tdSql.error(f"select * from t2 where c4 between null and {-1.7*10**308}") tdSql.query(f"select * from t2 where c4 between null and {-1.7*10**308}")
# tdSql.checkRows(3) # tdSql.checkRows(3)
tdLog.printNoPrefix("==========step8:query smallint type") tdLog.printNoPrefix("==========step8:query smallint type")
...@@ -151,7 +151,7 @@ class TDTestCase: ...@@ -151,7 +151,7 @@ class TDTestCase:
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.query("select * from t2 where c5 between -32768 and -32767") tdSql.query("select * from t2 where c5 between -32768 and -32767")
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.error("select * from t2 where c5 between null and -32767") tdSql.query("select * from t2 where c5 between null and -32767")
# tdSql.checkRows(1) # tdSql.checkRows(1)
tdLog.printNoPrefix("==========step9:query tinyint type") tdLog.printNoPrefix("==========step9:query tinyint type")
...@@ -170,21 +170,21 @@ class TDTestCase: ...@@ -170,21 +170,21 @@ class TDTestCase:
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.query("select * from t2 where c6 between -128 and -127") tdSql.query("select * from t2 where c6 between -128 and -127")
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.error("select * from t2 where c6 between null and -127") tdSql.query("select * from t2 where c6 between null and -127")
# tdSql.checkRows(3) # tdSql.checkRows(3)
tdLog.printNoPrefix("==========step10:invalid query type") tdLog.printNoPrefix("==========step10:invalid query type")
# tdSql.query("select * from supt where location between 'beijing' and 'shanghai'") tdSql.query("select * from supt where location between 'beijing' and 'shanghai'")
# tdSql.checkRows(23) tdSql.checkRows(23)
# # 非0值均解析为1，因此"between 负值 and o"解析为"between 1 and 0" # 非0值均解析为1，因此"between 负值 and o"解析为"between 1 and 0"
# tdSql.query("select * from supt where isused between 0 and 1") tdSql.query("select * from supt where isused between 0 and 1")
# tdSql.checkRows(23) tdSql.checkRows(23)
# tdSql.query("select * from supt where isused between -1 and 0") tdSql.query("select * from supt where isused between -1 and 0")
# tdSql.checkRows(0) tdSql.checkRows(0)
# tdSql.error("select * from supt where isused between false and true") tdSql.error("select * from supt where isused between false and true")
# tdSql.query("select * from supt where family between '拖拉机' and '自行车'") tdSql.query("select * from supt where family between '拖拉机' and '自行车'")
# tdSql.checkRows(23) tdSql.checkRows(23)
tdLog.printNoPrefix("==========step11:query HEX/OCT/BIN type") tdLog.printNoPrefix("==========step11:query HEX/OCT/BIN type")
......
tests/system-test/7-tmq/basic5.py
浏览文件 @ eb016e01
...@@ -112,18 +112,9 @@ class TDTestCase: ...@@ -112,18 +112,9 @@ class TDTestCase:
parameterDict["startTs"]) parameterDict["startTs"])
return return
def run(self):
tdSql.prepare()
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
cfgPath = buildPath + "/../sim/psim/cfg"
tdLog.info("cfgPath: %s" % cfgPath)
tdLog.printNoPrefix("======== test scenario 1: ") def tmqCase1(self, cfgPath, buildPath):
tdLog.printNoPrefix("======== test case 1: ")
tdLog.info("step 1: create database, stb, ctb and insert data") tdLog.info("step 1: create database, stb, ctb and insert data")
# create and start thread # create and start thread
parameterDict = {'cfg': '', \ parameterDict = {'cfg': '', \
...@@ -131,8 +122,8 @@ class TDTestCase: ...@@ -131,8 +122,8 @@ class TDTestCase:
'vgroups': 1, \ 'vgroups': 1, \
'stbName': 'stb', \ 'stbName': 'stb', \
'ctbNum': 10, \ 'ctbNum': 10, \
'rowsPerTbl': 10000, \ 'rowsPerTbl': 10, \
'batchNum': 10, \ 'batchNum': 200, \
'startTs': 1640966400000} # 2022-01-01 00:00:00.000 'startTs': 1640966400000} # 2022-01-01 00:00:00.000
parameterDict['cfg'] = cfgPath parameterDict['cfg'] = cfgPath
prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
...@@ -141,10 +132,7 @@ class TDTestCase: ...@@ -141,10 +132,7 @@ class TDTestCase:
# wait stb ready # wait stb ready
while 1: while 1:
#tdSql.query("show %s.stables"%parameterDict['dbName']) tdSql.query("show %s.stables"%parameterDict['dbName'])
tdSql.query("show db.stables")
#print (self.queryResult)
#print (tdSql.getRows())
if tdSql.getRows() == 1: if tdSql.getRows() == 1:
break break
else: else:
...@@ -159,32 +147,23 @@ class TDTestCase: ...@@ -159,32 +147,23 @@ class TDTestCase:
time.sleep(1) time.sleep(1)
tdSql.query("show topics") tdSql.query("show topics")
print ("======================================")
#print (self.queryResult)
#tdSql.checkRows(2) #tdSql.checkRows(2)
topic1 = tdSql.getData(0 , 0) topic1 = tdSql.getData(0 , 0)
topic2 = tdSql.getData(1 , 0) topic2 = tdSql.getData(1 , 0)
print (topic1)
print (topic2)
print (topicFromStb) tdLog.info("show topics: %s, %s"%(topic1, topic2))
print (topicFromCtb) if topic1 != topicFromStb and topic1 != topicFromCtb:
#tdLog.info("show topics: %s, %s"%topic1, topic2) tdLog.exit("topic error1")
#if topic1 != topicFromStb or topic1 != topicFromCtb: if topic2 != topicFromStb and topic2 != topicFromCtb:
# tdLog.exit("topic error1") tdLog.exit("topic error2")
#if topic2 != topicFromStb or topic2 != topicFromCtb:
# tdLog.exit("topic error2")
tdLog.info("create consume info table and consume result table") tdLog.info("create consume info table and consume result table")
cdbName = parameterDict["dbName"] cdbName = parameterDict["dbName"]
#tdSql.query("create database %s"%cdbName)
#tdSql.query("use %s"%cdbName)
tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)") tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)")
tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)") tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)")
consumerId = 0 consumerId = 0
expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"] expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"]
topicList = topicFromStb topicList = topicFromStb
ifcheckdata = 0 ifcheckdata = 0
keyList = 'group.id:cgrp1,\ keyList = 'group.id:cgrp1,\
...@@ -192,7 +171,7 @@ class TDTestCase: ...@@ -192,7 +171,7 @@ class TDTestCase:
auto.commit.interval.ms:6000,\ auto.commit.interval.ms:6000,\
auto.offset.reset:earliest' auto.offset.reset:earliest'
sql = "insert into consumeinfo values " sql = "insert into consumeinfo values "
sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata) sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
tdSql.query(sql) tdSql.query(sql)
tdLog.info("check stb if there are data") tdLog.info("check stb if there are data")
...@@ -229,19 +208,152 @@ class TDTestCase: ...@@ -229,19 +208,152 @@ class TDTestCase:
else: else:
time.sleep(5) time.sleep(5)
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] tdLog.info("consumer result: %d, %d"%(tdSql.getData(0 , 2), tdSql.getData(0 , 3)))
tdSql.checkData(0 , 1, consumerId)
# mulit rows and mulit tables in one sql, this num of msg is not sure
#tdSql.checkData(0 , 2, expectmsgcnt)
tdSql.checkData(0 , 3, expectrowcnt)
tdSql.query("drop topic %s"%topicFromStb)
tdSql.query("drop topic %s"%topicFromCtb)
tdLog.printNoPrefix("======== test case 1 end ...... ")
def tmqCase2(self, cfgPath, buildPath):
tdLog.printNoPrefix("======== test case 2: add child table with consuming ")
# create and start thread
parameterDict = {'cfg': '', \
'dbName': 'db2', \
'vgroups': 1, \
'stbName': 'stb', \
'ctbNum': 10, \
'rowsPerTbl': 10000, \
'batchNum': 100, \
'startTs': 1640966400000} # 2022-01-01 00:00:00.000
parameterDict['cfg'] = cfgPath
prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
prepareEnvThread.start()
# wait db ready
while 1:
tdSql.query("show databases")
if tdSql.getRows() == 4:
print (tdSql.getData(0,0), tdSql.getData(1,0),tdSql.getData(2,0),)
break
else:
time.sleep(1)
tdSql.query("use %s"%parameterDict['dbName'])
# wait stb ready
while 1:
tdSql.query("show %s.stables"%parameterDict['dbName'])
if tdSql.getRows() == 1:
break
else:
time.sleep(1)
tdLog.info("create topics from super table")
topicFromStb = 'topic_stb_column2'
topicFromCtb = 'topic_ctb_column2'
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName']))
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName']))
time.sleep(1)
tdSql.query("show topics")
topic1 = tdSql.getData(0 , 0)
topic2 = tdSql.getData(1 , 0)
tdLog.info("show topics: %s, %s"%(topic1, topic2))
if topic1 != topicFromStb and topic1 != topicFromCtb:
tdLog.exit("topic error1")
if topic2 != topicFromStb and topic2 != topicFromCtb:
tdLog.exit("topic error2")
tdLog.info("create consume info table and consume result table")
cdbName = parameterDict["dbName"]
tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
rowsOfNewCtb = 1000
consumerId = 0
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + rowsOfNewCtb
topicList = topicFromStb
ifcheckdata = 0
keyList = 'group.id:cgrp1,\
enable.auto.commit:false,\
auto.commit.interval.ms:6000,\
auto.offset.reset:earliest'
sql = "insert into consumeinfo values "
sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata)
tdSql.query(sql)
tdLog.info("check stb if there are data")
while 1:
tdSql.query("select count(*) from %s"%parameterDict["stbName"])
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
countOfStb = tdSql.getData(0, 0)
if countOfStb != 0:
tdLog.info("count from stb: %d"%countOfStb)
break
else:
time.sleep(1)
tdLog.info("start consume processor")
pollDelay = 5
showMsg = 1
showRow = 1
shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName)
shellCmd += "> /dev/null 2>&1 &"
tdLog.info(shellCmd)
os.system(shellCmd)
# create new child table and insert data
newCtbName = 'newctb'
tdSql.query("create table %s.%s using %s.%s tags(9999)"%(parameterDict["dbName"], newCtbName, parameterDict["dbName"], parameterDict["stbName"]))
startTs = parameterDict["startTs"]
for j in range(rowsOfNewCtb):
sql = "insert into %s.%s values (%d, %d, 'tmqrow_%d') "%(parameterDict["dbName"], newCtbName, startTs + j, j, j)
tdSql.execute(sql)
tdLog.debug("insert data into new child table ............ [OK]")
# wait for data ready
prepareEnvThread.join()
tdLog.info("insert process end, and start to check consume result")
while 1:
tdSql.query("select * from consumeresult")
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
if tdSql.getRows() == 1:
break
else:
time.sleep(5)
tdSql.checkData(0 , 1, consumerId) tdSql.checkData(0 , 1, consumerId)
tdSql.checkData(0 , 2, expectmsgcnt)
tdSql.checkData(0 , 3, expectrowcnt) tdSql.checkData(0 , 3, expectrowcnt)
tdLog.printNoPrefix("======== test scenario 2: ") tdSql.query("drop topic %s"%topicFromStb)
tdSql.query("drop topic %s"%topicFromCtb)
tdLog.printNoPrefix("======== test case 2 end ...... ")
tdLog.printNoPrefix("======== test scenario 3: ") def run(self):
tdSql.prepare()
#os.system('pkill tmq_sim') buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
cfgPath = buildPath + "/../sim/psim/cfg"
tdLog.info("cfgPath: %s" % cfgPath)
self.tmqCase1(cfgPath, buildPath)
#self.tmqCase2(cfgPath, buildPath)
#self.tmqCase3(cfgPath, buildPath)
def stop(self): def stop(self):
tdSql.close() tdSql.close()
......
tests/test/c/tmqSim.c
浏览文件 @ eb016e01
...@@ -333,8 +333,8 @@ void loop_consume(SThreadInfo* pInfo) { ...@@ -333,8 +333,8 @@ void loop_consume(SThreadInfo* pInfo) {
totalMsgs++; totalMsgs++;
if (totalMsgs >= pInfo->expectMsgCnt) { if (totalRows >= pInfo->expectMsgCnt) {
taosFprintfFile(g_fp, "==== totalMsgs >= pInfo->expectMsgCnt, so break\n"); taosFprintfFile(g_fp, "==== totalRows >= pInfo->expectMsgCnt, so break\n");
break; break;
} }
} else { } else {
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或