提交 49ff6bd8 编写于 作者: C Cary Xu

Merge branch '3.0' into feature/TD-11274-3.0

...@@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); ...@@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
typedef struct TAOS_MULTI_BIND { typedef struct TAOS_MULTI_BIND {
int buffer_type; int buffer_type;
void * buffer; void *buffer;
uintptr_t buffer_length; uintptr_t buffer_length;
int32_t * length; int32_t *length;
char * is_null; char *is_null;
int num; int num;
} TAOS_MULTI_BIND; } TAOS_MULTI_BIND;
...@@ -130,7 +130,7 @@ DLL_EXPORT void taos_cleanup(void); ...@@ -130,7 +130,7 @@ DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT setConfRet taos_set_config(const char *config);
DLL_EXPORT int taos_init(void); DLL_EXPORT int taos_init(void);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
const char *db, int dbLen, uint16_t port); const char *db, int dbLen, uint16_t port);
DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
...@@ -147,17 +147,17 @@ DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name ...@@ -147,17 +147,17 @@ DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name
DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt);
DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt);
...@@ -179,11 +179,11 @@ DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); ...@@ -179,11 +179,11 @@ DLL_EXPORT bool taos_is_update_query(TAOS_RES *res);
DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
DLL_EXPORT int * taos_get_column_data_offset(TAOS_RES *res, int columnIndex); DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
DLL_EXPORT void taos_reset_current_db(TAOS *taos); DLL_EXPORT void taos_reset_current_db(TAOS *taos);
DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);
DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
...@@ -204,17 +204,17 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); ...@@ -204,17 +204,17 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
#endif #endif
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
/* --------------------------TMQ INTERFACE------------------------------- */ /* --------------------------TMQ INTERFACE------------------------------- */
enum tmq_resp_err_t { enum {
TMQ_RESP_ERR__FAIL = -1, TMQ_RESP_ERR__FAIL = -1,
TMQ_RESP_ERR__SUCCESS = 0, TMQ_RESP_ERR__SUCCESS = 0,
}; };
typedef enum tmq_resp_err_t tmq_resp_err_t; typedef int32_t tmq_resp_err_t;
typedef struct tmq_t tmq_t; typedef struct tmq_t tmq_t;
typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t;
...@@ -229,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new(); ...@@ -229,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
DLL_EXPORT void tmq_list_destroy(tmq_list_t *); DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *); DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *);
DLL_EXPORT char ** tmq_list_to_c_array(const tmq_list_t *); DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *);
DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
...@@ -240,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); ...@@ -240,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
DLL_EXPORT TAOS_RES * tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
...@@ -260,7 +260,7 @@ enum tmq_conf_res_t { ...@@ -260,7 +260,7 @@ enum tmq_conf_res_t {
typedef enum tmq_conf_res_t tmq_conf_res_t; typedef enum tmq_conf_res_t tmq_conf_res_t;
DLL_EXPORT tmq_conf_t * tmq_conf_new(); DLL_EXPORT tmq_conf_t *tmq_conf_new();
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param); DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
......
...@@ -206,7 +206,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 ...@@ -206,7 +206,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
size_t blockDataGetSize(const SSDataBlock* pBlock); size_t blockDataGetSize(const SSDataBlock* pBlock);
size_t blockDataGetRowSize(SSDataBlock* pBlock); size_t blockDataGetRowSize(SSDataBlock* pBlock);
double blockDataGetSerialRowSize(const SSDataBlock* pBlock); double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock); size_t blockDataGetSerialMetaSize(uint32_t numOfCols);
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
...@@ -238,7 +238,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool ...@@ -238,7 +238,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
const char* stbFullName, int32_t vgId); const char* stbFullName, int32_t vgId);
static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock); return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock);
} }
static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,
......
...@@ -2500,6 +2500,7 @@ typedef struct { ...@@ -2500,6 +2500,7 @@ typedef struct {
int64_t sliding; int64_t sliding;
int64_t dstTbUid; int64_t dstTbUid;
int32_t dstVgId; // for stream int32_t dstVgId; // for stream
SEpSet epSet;
char* expr; char* expr;
} STableIndexInfo; } STableIndexInfo;
......
...@@ -71,16 +71,21 @@ typedef struct SCatalogReq { ...@@ -71,16 +71,21 @@ typedef struct SCatalogReq {
bool forceUpdate; bool forceUpdate;
} SCatalogReq; } SCatalogReq;
typedef struct SMetaRes {
int32_t code;
void* pRes;
} SMetaRes;
typedef struct SMetaData { typedef struct SMetaData {
SArray *pDbVgroup; // SArray<SArray<SVgroupInfo>*> SArray *pDbVgroup; // pRes = SArray<SVgroupInfo>*
SArray *pDbCfg; // SArray<SDbCfgInfo> SArray *pDbCfg; // pRes = SDbCfgInfo*
SArray *pDbInfo; // SArray<SDbInfo> SArray *pDbInfo; // pRes = SDbInfo*
SArray *pTableMeta; // SArray<STableMeta*> SArray *pTableMeta; // pRes = STableMeta*
SArray *pTableHash; // SArray<SVgroupInfo> SArray *pTableHash; // pRes = SVgroupInfo*
SArray *pUdfList; // SArray<SFuncInfo> SArray *pUdfList; // pRes = SFuncInfo*
SArray *pIndex; // SArray<SIndexInfo> SArray *pIndex; // pRes = SIndexInfo*
SArray *pUser; // SArray<bool> SArray *pUser; // pRes = bool*
SArray *pQnodeList; // SArray<SQueryNodeAddr> SArray *pQnodeList; // pRes = SQueryNodeAddr*
} SMetaData; } SMetaData;
typedef struct SCatalogCfg { typedef struct SCatalogCfg {
......
...@@ -197,6 +197,7 @@ typedef struct SAggFunctionInfo { ...@@ -197,6 +197,7 @@ typedef struct SAggFunctionInfo {
struct SScalarParam { struct SScalarParam {
SColumnInfoData *columnData; SColumnInfoData *columnData;
SHashObj *pHashFilter; SHashObj *pHashFilter;
int32_t hashValueType;
void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value
int32_t numOfRows; int32_t numOfRows;
}; };
......
...@@ -122,6 +122,14 @@ typedef enum EFunctionType { ...@@ -122,6 +122,14 @@ typedef enum EFunctionType {
// internal function // internal function
FUNCTION_TYPE_SELECT_VALUE, FUNCTION_TYPE_SELECT_VALUE,
// distributed splitting functions
FUNCTION_TYPE_APERCENTILE_PARTIAL,
FUNCTION_TYPE_APERCENTILE_MERGE,
FUNCTION_TYPE_SPREAD_PARTIAL,
FUNCTION_TYPE_SPREAD_MERGE,
FUNCTION_TYPE_HISTOGRAM_PARTIAL,
FUNCTION_TYPE_HISTOGRAM_MERGE,
// user defined funcion // user defined funcion
FUNCTION_TYPE_UDF = 10000 FUNCTION_TYPE_UDF = 10000
} EFunctionType; } EFunctionType;
......
...@@ -210,8 +210,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t ...@@ -210,8 +210,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ #define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \
((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \ ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \
(_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \ (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \
(_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code == TSDB_CODE_PAR_VALUE_TOO_LONG) || \ (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
(_code == TSDB_CODE_PAR_INVALID_DROP_COL)) (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
......
...@@ -33,7 +33,7 @@ SSyncNode* syncNodeAcquire(int64_t rid); ...@@ -33,7 +33,7 @@ SSyncNode* syncNodeAcquire(int64_t rid);
void syncNodeRelease(SSyncNode* pNode); void syncNodeRelease(SSyncNode* pNode);
int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg); int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg); int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo);
void syncSetMsgCb(int64_t rid, const SMsgCb* msgcb); void syncSetMsgCb(int64_t rid, const SMsgCb* msgcb);
char* sync2SimpleStr(int64_t rid); char* sync2SimpleStr(int64_t rid);
......
...@@ -33,7 +33,10 @@ extern int32_t tsRpcHeadSize; ...@@ -33,7 +33,10 @@ extern int32_t tsRpcHeadSize;
typedef struct { typedef struct {
uint32_t clientIp; uint32_t clientIp;
uint16_t clientPort; uint16_t clientPort;
char user[TSDB_USER_LEN]; union {
char user[TSDB_USER_LEN];
int64_t applyIndex;
};
} SRpcConnInfo; } SRpcConnInfo;
typedef struct SRpcHandleInfo { typedef struct SRpcHandleInfo {
......
...@@ -65,7 +65,7 @@ void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ ...@@ -65,7 +65,7 @@ void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __
* @param flags * @param flags
* @return * @return
*/ */
void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t fn, int32_t flags); void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size, __compar_fn_t compar, int32_t flags);
/** /**
* adjust heap * adjust heap
...@@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size ...@@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
* @return * @return
*/ */
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char* buf, bool maxroot); __ext_compar_fn_t compar, char *buf, bool maxroot);
/** /**
* sort heap to make sure it is a max/min root heap * sort heap to make sure it is a max/min root heap
...@@ -97,8 +97,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const ...@@ -97,8 +97,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
* @param maxroot: if heap is max root heap * @param maxroot: if heap is max root heap
* @return * @return
*/ */
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot);
bool maxroot);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -339,9 +339,9 @@ int32_t* taosGetErrno(); ...@@ -339,9 +339,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060D) #define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060D)
#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060E) #define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060E)
#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060F) #define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060F)
#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x0600) #define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x0610)
#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x0601) #define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x0611)
#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x0602) #define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x0612)
#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0613) #define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0613)
#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0614) #define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0614)
#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0615)
......
...@@ -32,7 +32,7 @@ extern "C" { ...@@ -32,7 +32,7 @@ extern "C" {
#define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle #define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle
// Bytes for each type. // Bytes for each type.
extern const int32_t TYPE_BYTES[15]; extern const int32_t TYPE_BYTES[16];
// TODO: replace and remove code below // TODO: replace and remove code below
#define CHAR_BYTES sizeof(char) #define CHAR_BYTES sizeof(char)
...@@ -356,8 +356,8 @@ typedef enum ELogicConditionType { ...@@ -356,8 +356,8 @@ typedef enum ELogicConditionType {
#define TSDB_DEFAULT_EXPLAIN_VERBOSE false #define TSDB_DEFAULT_EXPLAIN_VERBOSE false
#define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024 #define TSDB_EXPLAIN_RESULT_ROW_SIZE 512
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN" #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
#define TSDB_MAX_FIELD_LEN 16384 #define TSDB_MAX_FIELD_LEN 16384
#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 #define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384
......
...@@ -45,7 +45,7 @@ extern "C" { ...@@ -45,7 +45,7 @@ extern "C" {
#define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
#define HEARTBEAT_INTERVAL 1500 // ms #define HEARTBEAT_INTERVAL 1500 // ms
#define SYNC_ON_TOP_OF_ASYNC 0 #define SYNC_ON_TOP_OF_ASYNC 1
enum { enum {
RES_TYPE__QUERY = 1, RES_TYPE__QUERY = 1,
...@@ -144,7 +144,7 @@ typedef struct STscObj { ...@@ -144,7 +144,7 @@ typedef struct STscObj {
int32_t numOfReqs; // number of sqlObj bound to this connection int32_t numOfReqs; // number of sqlObj bound to this connection
SAppInstInfo* pAppInfo; SAppInstInfo* pAppInfo;
SHashObj* pRequests; SHashObj* pRequests;
int8_t schemalessType; int8_t schemalessType; // todo remove it, this attribute should be move to request
} STscObj; } STscObj;
typedef struct SResultColumn { typedef struct SResultColumn {
......
...@@ -580,7 +580,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { ...@@ -580,7 +580,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) { if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code),
pRequest->retry, pRequest->requestId);
pRequest->prevCode = code; pRequest->prevCode = code;
doAsyncQuery(pRequest, true); doAsyncQuery(pRequest, true);
return; return;
...@@ -592,6 +593,7 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { ...@@ -592,6 +593,7 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
pRequest->code = code; pRequest->code = code;
} }
tscDebug("schedulerExecCb request type %s", TMSG_INFO(pRequest->type));
if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
removeMeta(pTscObj, pRequest->tableList); removeMeta(pTscObj, pRequest->tableList);
} }
...@@ -695,6 +697,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { ...@@ -695,6 +697,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob, schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob,
pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest); pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest);
} else {
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
} }
//todo not to be released here //todo not to be released here
...@@ -702,7 +706,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { ...@@ -702,7 +706,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
break; break;
} }
case QUERY_EXEC_MODE_EMPTY_RESULT: case QUERY_EXEC_MODE_EMPTY_RESULT:
pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
break; break;
default: default:
break; break;
...@@ -1280,8 +1285,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int ...@@ -1280,8 +1285,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
varDataVal(dst) + CHAR_BYTES); varDataVal(dst) + CHAR_BYTES);
if (length <= 0) { if (length <= 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
varDataVal(jsonInnerData));
length = 0; length = 0;
} }
varDataSetLen(dst, length + CHAR_BYTES * 2); varDataSetLen(dst, length + CHAR_BYTES * 2);
...@@ -1340,6 +1344,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 ...@@ -1340,6 +1344,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
uint64_t groupId = *(uint64_t*)p; uint64_t groupId = *(uint64_t*)p;
p += sizeof(uint64_t); p += sizeof(uint64_t);
// check fields
for(int32_t i = 0; i < numOfCols; ++i) {
int16_t type = *(int16_t*) p;
p += sizeof(int16_t);
int32_t bytes = *(int32_t*) p;
p += sizeof(int32_t);
// ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);
}
int32_t* colLength = (int32_t*)p; int32_t* colLength = (int32_t*)p;
p += sizeof(int32_t) * numOfCols; p += sizeof(int32_t) * numOfCols;
......
...@@ -500,7 +500,11 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { ...@@ -500,7 +500,11 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
return 0; return 0;
} }
doFetchRows(pRequest, false, true); #if SYNC_ON_TOP_OF_ASYNC
doAsyncFetchRow(pRequest, false, true);
#else
doFetchRows(pRequest, true, true);
#endif
// TODO refactor // TODO refactor
SReqResultInfo *pResultInfo = &pRequest->body.resInfo; SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
...@@ -625,8 +629,10 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) { ...@@ -625,8 +629,10 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) {
taosMemoryFree(pWrapper); taosMemoryFree(pWrapper);
launchAsyncQuery(pRequest, pQuery); launchAsyncQuery(pRequest, pQuery);
} else { } else {
tscDebug("error happens, code:%d", code);
if (NEED_CLIENT_HANDLE_ERROR(code)) { if (NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code),
pRequest->retry, pRequest->requestId);
pRequest->prevCode = code; pRequest->prevCode = code;
doAsyncQuery(pRequest, true); doAsyncQuery(pRequest, true);
return; return;
...@@ -691,6 +697,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) { ...@@ -691,6 +697,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
.pTransporter = pTscObj->pAppInfo->pTransporter, .pTransporter = pTscObj->pAppInfo->pTransporter,
.pStmtCb = NULL, .pStmtCb = NULL,
.pUser = pTscObj->user, .pUser = pTscObj->user,
.schemalessType = pTscObj->schemalessType,
.isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
.async = true,}; .async = true,};
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -699,13 +706,14 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) { ...@@ -699,13 +706,14 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) { void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
SParseContext* pCxt = NULL; SParseContext* pCxt = NULL;
STscObj *pTscObj = pRequest->pTscObj; STscObj *pTscObj = pRequest->pTscObj;
int32_t code = 0;
if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) { if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) {
pRequest->code = pRequest->prevCode; code = pRequest->prevCode;
goto _error; goto _error;
} }
int32_t code = createParseContext(pRequest, &pCxt); code = createParseContext(pRequest, &pCxt);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
...@@ -742,7 +750,7 @@ void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) { ...@@ -742,7 +750,7 @@ void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
} }
_error: _error:
tscError("0x%"PRIx64" error happens, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); tscError("0x%"PRIx64" error happens, code:%d - %s, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId);
terrno = code; terrno = code;
pRequest->code = code; pRequest->code = code;
pRequest->body.queryFp(pRequest->body.param, pRequest, code); pRequest->body.queryFp(pRequest->body.param, pRequest, code);
......
...@@ -323,7 +323,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { ...@@ -323,7 +323,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; pParam->rspErr = code;
if (pParam->async) { if (pParam->async) {
if (pParam->automatic && pParam->tmq->commitCb) { if (pParam->automatic && pParam->tmq->commitCb) {
pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets, pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets,
...@@ -432,12 +432,13 @@ int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_ ...@@ -432,12 +432,13 @@ int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_
code = pParam->rspErr; code = pParam->rspErr;
tsem_destroy(&pParam->rspSem); tsem_destroy(&pParam->rspSem);
taosMemoryFree(pParam); taosMemoryFree(pParam);
} else {
code = 0;
} }
// avoid double free if msg is sent // avoid double free if msg is sent
buf = NULL; buf = NULL;
code = 0;
END: END:
if (buf) taosMemoryFree(buf); if (buf) taosMemoryFree(buf);
/*if (pParam) taosMemoryFree(pParam);*/ /*if (pParam) taosMemoryFree(pParam);*/
...@@ -445,9 +446,9 @@ END: ...@@ -445,9 +446,9 @@ END:
if (code != 0 && async) { if (code != 0 && async) {
if (automatic) { if (automatic) {
tmq->commitCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam); tmq->commitCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
} else { } else {
userCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, userParam); userCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
} }
} }
...@@ -1474,16 +1475,16 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { ...@@ -1474,16 +1475,16 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
if (tmq->status == TMQ_CONSUMER_STATUS__READY) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL); tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL);
if (rsp == TMQ_RESP_ERR__FAIL) { if (rsp != TMQ_RESP_ERR__SUCCESS) {
return TMQ_RESP_ERR__FAIL; return rsp;
} }
tmq_list_t* lst = tmq_list_new(); tmq_list_t* lst = tmq_list_new();
rsp = tmq_subscribe(tmq, lst); rsp = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst); tmq_list_destroy(lst);
if (rsp == TMQ_RESP_ERR__FAIL) { if (rsp != TMQ_RESP_ERR__SUCCESS) {
return TMQ_RESP_ERR__FAIL; return rsp;
} }
} }
// TODO: free resources // TODO: free resources
...@@ -1493,8 +1494,11 @@ tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { ...@@ -1493,8 +1494,11 @@ tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
const char* tmq_err2str(tmq_resp_err_t err) { const char* tmq_err2str(tmq_resp_err_t err) {
if (err == TMQ_RESP_ERR__SUCCESS) { if (err == TMQ_RESP_ERR__SUCCESS) {
return "success"; return "success";
} else if (err == TMQ_RESP_ERR__FAIL) {
return "fail";
} else {
return tstrerror(err);
} }
return "fail";
} }
const char* tmq_get_topic_name(TAOS_RES* res) { const char* tmq_get_topic_name(TAOS_RES* res) {
......
...@@ -780,31 +780,28 @@ TEST(testCase, async_api_test) { ...@@ -780,31 +780,28 @@ TEST(testCase, async_api_test) {
taos_query(pConn, "use test"); taos_query(pConn, "use test");
TAOS_RES* pRes = taos_query(pConn, "select * from t1"); TAOS_RES* pRes = taos_query(pConn, "desc abc1.tu");
taos_query(pConn, "alter table t1 add column b int");
pRes = taos_query(pConn, "insert into t1 values(now, 1, 2)");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed, reason:%s\n", taos_errstr(pRes)); printf("failed, reason:%s\n", taos_errstr(pRes));
} }
// int32_t n = 0; int32_t n = 0;
// TAOS_ROW pRow = NULL; TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes); int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0}; char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) { while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t* length = taos_fetch_lengths(pRes); int32_t* length = taos_fetch_lengths(pRes);
// for(int32_t i = 0; i < numOfFields; ++i) { for(int32_t i = 0; i < numOfFields; ++i) {
// printf("(%d):%d " , i, length[i]); printf("(%d):%d " , i, length[i]);
// } }
// printf("\n"); printf("\n");
//
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str); printf("%s\n", str);
// memset(str, 0, sizeof(str)); memset(str, 0, sizeof(str));
// } }
taos_query_a(pConn, "alter table test.m1 comment 'abcde' ", queryCallback, pConn); taos_query_a(pConn, "alter table test.m1 comment 'abcde' ", queryCallback, pConn);
getchar(); getchar();
......
...@@ -682,9 +682,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { ...@@ -682,9 +682,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
* @param pBlock * @param pBlock
* @return * @return
*/ */
size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { size_t blockDataGetSerialMetaSize(uint32_t numOfCols) {
// | total rows/total length | block group id | each column length | // | total rows/total length | block group id | column schema | each column length |
return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t); return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t);
} }
double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
...@@ -1217,6 +1217,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { ...@@ -1217,6 +1217,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
pBlock->info.numOfCols = numOfCols; pBlock->info.numOfCols = numOfCols;
pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
pBlock->info.rowSize = pDataBlock->info.rowSize; pBlock->info.rowSize = pDataBlock->info.rowSize;
pBlock->info.groupId = pDataBlock->info.groupId;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = {0};
...@@ -1246,7 +1247,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { ...@@ -1246,7 +1247,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
} }
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock); int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols);
int32_t rowSize = pBlock->info.rowSize; int32_t rowSize = pBlock->info.rowSize;
...@@ -1271,12 +1272,12 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { ...@@ -1271,12 +1272,12 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
void colDataDestroy(SColumnInfoData* pColData) { void colDataDestroy(SColumnInfoData* pColData) {
if (IS_VAR_DATA_TYPE(pColData->info.type)) { if (IS_VAR_DATA_TYPE(pColData->info.type)) {
taosMemoryFree(pColData->varmeta.offset); taosMemoryFreeClear(pColData->varmeta.offset);
} else { } else {
taosMemoryFree(pColData->nullbitmap); taosMemoryFreeClear(pColData->nullbitmap);
} }
taosMemoryFree(pColData->pData); taosMemoryFreeClear(pColData->pData);
} }
static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
...@@ -1885,34 +1886,44 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo ...@@ -1885,34 +1886,44 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
int8_t needCompress) { int8_t needCompress) {
// todo extract method
int32_t* actualLen = (int32_t*)data; int32_t* actualLen = (int32_t*)data;
data += sizeof(int32_t); data += sizeof(int32_t);
uint64_t* groupId = (uint64_t*)data; uint64_t* groupId = (uint64_t*)data;
data += sizeof(uint64_t); data += sizeof(uint64_t);
for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
*((int16_t*) data) = pColInfoData->info.type;
data += sizeof(int16_t);
*((int32_t*) data) = pColInfoData->info.bytes;
data += sizeof(int32_t);
}
int32_t* colSizes = (int32_t*)data; int32_t* colSizes = (int32_t*)data;
data += numOfCols * sizeof(int32_t); data += numOfCols * sizeof(int32_t);
*dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t)); *dataLen = blockDataGetSerialMetaSize(numOfCols);
int32_t numOfRows = pBlock->info.rows; int32_t numOfRows = pBlock->info.rows;
for (int32_t col = 0; col < numOfCols; ++col) { for (int32_t col = 0; col < numOfCols; ++col) {
SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col); SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col);
// copy the null bitmap // copy the null bitmap
size_t metaSize = 0;
if (IS_VAR_DATA_TYPE(pColRes->info.type)) { if (IS_VAR_DATA_TYPE(pColRes->info.type)) {
size_t metaSize = numOfRows * sizeof(int32_t); metaSize = numOfRows * sizeof(int32_t);
memcpy(data, pColRes->varmeta.offset, metaSize); memcpy(data, pColRes->varmeta.offset, metaSize);
data += metaSize;
(*dataLen) += metaSize;
} else { } else {
int32_t len = BitmapLen(numOfRows); metaSize = BitmapLen(numOfRows);
memcpy(data, pColRes->nullbitmap, len); memcpy(data, pColRes->nullbitmap, metaSize);
data += len;
(*dataLen) += len;
} }
data += metaSize;
(*dataLen) += metaSize;
if (needCompress) { if (needCompress) {
colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress);
data += colSizes[col]; data += colSizes[col];
...@@ -1941,6 +1952,17 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t ...@@ -1941,6 +1952,17 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
pBlock->info.groupId = *(uint64_t*)pStart; pBlock->info.groupId = *(uint64_t*)pStart;
pStart += sizeof(uint64_t); pStart += sizeof(uint64_t);
for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
pColInfoData->info.type = *(int16_t*)pStart;
pStart += sizeof(int16_t);
pColInfoData->info.bytes = *(int32_t*)pStart;
pStart += sizeof(int32_t);
}
blockDataEnsureCapacity(pBlock, numOfRows);
int32_t* colLen = (int32_t*)pStart; int32_t* colLen = (int32_t*)pStart;
pStart += sizeof(int32_t) * numOfCols; pStart += sizeof(int32_t) * numOfCols;
......
...@@ -332,7 +332,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { ...@@ -332,7 +332,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
if (cfgAddString(pCfg, "smlTagNullName", tsSmlTagName, 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 4; tsNumOfTaskQueueThreads = tsNumOfCores / 4;
...@@ -532,7 +532,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { ...@@ -532,7 +532,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
} }
tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagNullName")->str, TSDB_COL_NAME_LEN); tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
......
...@@ -2427,6 +2427,7 @@ int32_t tSerializeSTableIndexInfo(SEncoder *pEncoder, STableIndexInfo* pInfo) { ...@@ -2427,6 +2427,7 @@ int32_t tSerializeSTableIndexInfo(SEncoder *pEncoder, STableIndexInfo* pInfo) {
if (tEncodeI64(pEncoder, pInfo->sliding) < 0) return -1; if (tEncodeI64(pEncoder, pInfo->sliding) < 0) return -1;
if (tEncodeI64(pEncoder, pInfo->dstTbUid) < 0) return -1; if (tEncodeI64(pEncoder, pInfo->dstTbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pInfo->dstVgId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->dstVgId) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1;
if (tEncodeCStr(pEncoder, pInfo->expr) < 0) return -1; if (tEncodeCStr(pEncoder, pInfo->expr) < 0) return -1;
return 0; return 0;
} }
...@@ -2459,6 +2460,7 @@ int32_t tDeserializeSTableIndexInfo(SDecoder *pDecoder, STableIndexInfo *pInfo) ...@@ -2459,6 +2460,7 @@ int32_t tDeserializeSTableIndexInfo(SDecoder *pDecoder, STableIndexInfo *pInfo)
if (tDecodeI64(pDecoder, &pInfo->sliding) < 0) return -1; if (tDecodeI64(pDecoder, &pInfo->sliding) < 0) return -1;
if (tDecodeI64(pDecoder, &pInfo->dstTbUid) < 0) return -1; if (tDecodeI64(pDecoder, &pInfo->dstTbUid) < 0) return -1;
if (tDecodeI32(pDecoder, &pInfo->dstVgId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->dstVgId) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pInfo->expr) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pInfo->expr) < 0) return -1;
return 0; return 0;
......
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
#include "tcompression.h" #include "tcompression.h"
#include "trow.h" #include "trow.h"
const int32_t TYPE_BYTES[15] = { const int32_t TYPE_BYTES[16] = {
-1, // TSDB_DATA_TYPE_NULL -1, // TSDB_DATA_TYPE_NULL
CHAR_BYTES, // TSDB_DATA_TYPE_BOOL CHAR_BYTES, // TSDB_DATA_TYPE_BOOL
CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT
...@@ -34,6 +34,7 @@ const int32_t TYPE_BYTES[15] = { ...@@ -34,6 +34,7 @@ const int32_t TYPE_BYTES[15] = {
SHORT_BYTES, // TSDB_DATA_TYPE_USMALLINT SHORT_BYTES, // TSDB_DATA_TYPE_USMALLINT
INT_BYTES, // TSDB_DATA_TYPE_UINT INT_BYTES, // TSDB_DATA_TYPE_UINT
sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT
TSDB_MAX_JSON_TAG_LEN, // TSDB_DATA_TYPE_JSON
}; };
#define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \ #define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \
......
...@@ -34,6 +34,7 @@ typedef struct SVnodeMgmt { ...@@ -34,6 +34,7 @@ typedef struct SVnodeMgmt {
SQWorkerPool fetchPool; SQWorkerPool fetchPool;
SWWorkerPool syncPool; SWWorkerPool syncPool;
SWWorkerPool writePool; SWWorkerPool writePool;
SWWorkerPool applyPool;
SWWorkerPool mergePool; SWWorkerPool mergePool;
SSingleWorker mgmtWorker; SSingleWorker mgmtWorker;
SSingleWorker monitorWorker; SSingleWorker monitorWorker;
...@@ -52,10 +53,9 @@ typedef struct { ...@@ -52,10 +53,9 @@ typedef struct {
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int32_t refCount;
int32_t vgVersion; int32_t vgVersion;
int32_t refCount;
int8_t dropped; int8_t dropped;
int8_t accessState;
char *path; char *path;
SVnode *pImpl; SVnode *pImpl;
STaosQueue *pWriteQ; STaosQueue *pWriteQ;
......
...@@ -323,7 +323,6 @@ SArray *vmGetMsgHandles() { ...@@ -323,7 +323,6 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MON_VM_INFO, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MON_VM_INFO, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
// Requests handled by VNODE
if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
......
...@@ -49,10 +49,9 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { ...@@ -49,10 +49,9 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
} }
pVnode->vgId = pCfg->vgId; pVnode->vgId = pCfg->vgId;
pVnode->refCount = 0;
pVnode->vgVersion = pCfg->vgVersion; pVnode->vgVersion = pCfg->vgVersion;
pVnode->refCount = 0;
pVnode->dropped = 0; pVnode->dropped = 0;
pVnode->accessState = TSDB_VN_ALL_ACCCESS;
pVnode->path = tstrdup(pCfg->path); pVnode->path = tstrdup(pCfg->path);
pVnode->pImpl = pImpl; pVnode->pImpl = pImpl;
...@@ -96,7 +95,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { ...@@ -96,7 +95,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
dDebug("vgId:%d, vnode is closed", pVnode->vgId); dDebug("vgId:%d, vnode is closed", pVnode->vgId);
if (pVnode->dropped) { if (pVnode->dropped) {
dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped); dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId); snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
vnodeDestroy(path, pMgmt->pTfs); vnodeDestroy(path, pMgmt->pTfs);
} }
......
...@@ -16,10 +16,8 @@ ...@@ -16,10 +16,8 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "vmInt.h" #include "vmInt.h"
#include "sync.h"
#include "syncTools.h"
static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) { static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
if (pMsg->info.handle == NULL) return;
SRpcMsg rsp = { SRpcMsg rsp = {
.code = code, .code = code,
.pCont = pMsg->info.rsp, .pCont = pMsg->info.rsp,
...@@ -53,9 +51,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { ...@@ -53,9 +51,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
} }
if (IsReq(pMsg)) { if (IsReq(pMsg)) {
if (code != 0 && terrno != 0) { if (code != 0) {
dError("msg:%p failed to process since %s", pMsg, terrstr()); if (terrno != 0) code = terrno;
code = terrno; dError("msg:%p, failed to process since %s", pMsg, terrstr());
} }
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
} }
...@@ -97,110 +95,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { ...@@ -97,110 +95,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
int32_t code = 0;
SRpcMsg *pMsg = NULL;
SVnodeObj *pVnode = pInfo->ahandle;
int64_t sync = vnodeGetSyncHandle(pVnode->pImpl);
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg **));
for (int32_t m = 0; m < numOfMsgs; m++) {
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
dTrace("vgId:%d, msg:%p get from vnode-write queue", pVnode->vgId, pMsg);
if (taosArrayPush(pArray, &pMsg) == NULL) {
dError("vgId:%d, failed to push msg:%p to vnode-write array", pVnode->vgId, pMsg);
vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
}
}
for (int32_t m = 0; m < taosArrayGetSize(pArray); m++) {
pMsg = *(SRpcMsg **)taosArrayGet(pArray, m);
code = vnodePreprocessReq(pVnode->pImpl, pMsg);
if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
dTrace("vgId:%d, msg:%p in progress and no rsp", pVnode->vgId, pMsg);
continue;
}
if (pMsg->msgType != TDMT_VND_ALTER_REPLICA) {
code = syncPropose(sync, pMsg, false);
}
if (code == TAOS_SYNC_PROPOSE_SUCCESS) {
dTrace("vgId:%d, msg:%p is proposed and no rsp", pVnode->vgId, pMsg);
continue;
} else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
SEpSet newEpSet = {0};
syncGetEpSet(sync, &newEpSet);
SEp *pEp = &newEpSet.eps[newEpSet.inUse];
if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
}
dTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", pVnode->vgId, pMsg,
newEpSet.numOfEps, newEpSet.inUse);
for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
dTrace("vgId:%d, msg:%p ep:%s:%u", pVnode->vgId, pMsg, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
}
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
tmsgSendRedirectRsp(&rsp, &newEpSet);
} else {
dError("vgId:%d, msg:%p failed to propose write since %s, code:0x%x", pVnode->vgId, pMsg, tstrerror(code), code);
vmSendRsp(pMsg, code);
}
}
for (int32_t i = 0; i < numOfMsgs; i++) {
pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
dTrace("vgId:%d, msg:%p is freed", pVnode->vgId, pMsg);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
taosArrayDestroy(pArray);
}
static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle;
SRpcMsg *pMsg = NULL;
for (int32_t i = 0; i < numOfMsgs; ++i) {
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
dTrace("vgId:%d, msg:%p get from vnode-apply queue", pVnode->vgId, pMsg);
// init response rpc msg
SRpcMsg rsp = {0};
// get original rpc msg
assert(pMsg->msgType == TDMT_SYNC_APPLY_MSG);
SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(pMsg);
syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
SRpcMsg originalRpcMsg;
syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
// apply data into tsdb
if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
rsp.code = terrno;
dError("vgId:%d, msg:%p failed to apply since %s", pVnode->vgId, pMsg, terrstr());
}
syncApplyMsgDestroy(pSyncApplyMsg);
rpcFreeCont(originalRpcMsg.pCont);
// if leader, send response
if (pMsg->info.handle != NULL) {
rsp.info = pMsg->info;
tmsgSendRsp(&rsp);
}
dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, rsp.code);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
}
static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
SRpcMsg *pMsg = NULL; SRpcMsg *pMsg = NULL;
...@@ -211,11 +105,9 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf ...@@ -211,11 +105,9 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL);
if (code != 0) { if (code != 0) {
if (terrno != 0) code = terrno;
dError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr()); dError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr());
if (pMsg->info.handle != NULL) { vmSendRsp(pMsg, code);
if (terrno != 0) code = terrno;
vmSendRsp(pMsg, code);
}
} }
dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
...@@ -234,8 +126,8 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO ...@@ -234,8 +126,8 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
if (code != 0) { if (code != 0) {
dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
} }
...@@ -254,7 +146,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp ...@@ -254,7 +146,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
dError("vgId:%d, failed to put msg:%p into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(), dError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
return terrno != 0 ? terrno : -1; return terrno != 0 ? terrno : -1;
} }
...@@ -322,7 +214,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { ...@@ -322,7 +214,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (pMsg == NULL) return -1; if (pMsg == NULL) return -1;
SMsgHead *pHead = pRpc->pCont; SMsgHead *pHead = pRpc->pCont;
dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pMsg->msgType)); dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pRpc->msgType));
pHead->contLen = htonl(pHead->contLen); pHead->contLen = htonl(pHead->contLen);
pHead->vgId = htonl(pHead->vgId); pHead->vgId = htonl(pHead->vgId);
...@@ -362,9 +254,9 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { ...@@ -362,9 +254,9 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
} }
int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeMsg);
pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyMsg);
pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue);
pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeQueue); pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeQueue);
...@@ -381,8 +273,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { ...@@ -381,8 +273,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
tWWorkerFreeQueue(&pMgmt->applyPool, pVnode->pApplyQ);
tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
tWWorkerFreeQueue(&pMgmt->mergePool, pVnode->pMergeQ); tWWorkerFreeQueue(&pMgmt->mergePool, pVnode->pMergeQ);
...@@ -413,6 +305,11 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { ...@@ -413,6 +305,11 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
pWPool->max = tsNumOfVnodeWriteThreads; pWPool->max = tsNumOfVnodeWriteThreads;
if (tWWorkerInit(pWPool) != 0) return -1; if (tWWorkerInit(pWPool) != 0) return -1;
SWWorkerPool *pAPool = &pMgmt->applyPool;
pAPool->name = "vnode-apply";
pAPool->max = tsNumOfVnodeWriteThreads;
if (tWWorkerInit(pAPool) != 0) return -1;
SWWorkerPool *pSPool = &pMgmt->syncPool; SWWorkerPool *pSPool = &pMgmt->syncPool;
pSPool->name = "vnode-sync"; pSPool->name = "vnode-sync";
pSPool->max = tsNumOfVnodeSyncThreads; pSPool->max = tsNumOfVnodeSyncThreads;
...@@ -449,6 +346,7 @@ void vmStopWorker(SVnodeMgmt *pMgmt) { ...@@ -449,6 +346,7 @@ void vmStopWorker(SVnodeMgmt *pMgmt) {
tSingleWorkerCleanup(&pMgmt->monitorWorker); tSingleWorkerCleanup(&pMgmt->monitorWorker);
tSingleWorkerCleanup(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->mgmtWorker);
tWWorkerCleanup(&pMgmt->writePool); tWWorkerCleanup(&pMgmt->writePool);
tWWorkerCleanup(&pMgmt->applyPool);
tWWorkerCleanup(&pMgmt->syncPool); tWWorkerCleanup(&pMgmt->syncPool);
tQWorkerCleanup(&pMgmt->queryPool); tQWorkerCleanup(&pMgmt->queryPool);
tQWorkerCleanup(&pMgmt->fetchPool); tQWorkerCleanup(&pMgmt->fetchPool);
......
...@@ -92,6 +92,15 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { ...@@ -92,6 +92,15 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
ASSERT(pConsumer); ASSERT(pConsumer);
mInfo("receive consumer lost msg, consumer id %ld, status %s", pLostMsg->consumerId,
mndConsumerStatusName(pConsumer->status));
if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
mndReleaseConsumer(pMnode, pConsumer);
return -1;
}
SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
pConsumerNew->updateType = CONSUMER_UPDATE__LOST; pConsumerNew->updateType = CONSUMER_UPDATE__LOST;
......
...@@ -266,7 +266,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { ...@@ -266,7 +266,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} }
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns +
blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock); blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols);
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {
......
...@@ -894,6 +894,15 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn ...@@ -894,6 +894,15 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn
info.sliding = pSma->sliding; info.sliding = pSma->sliding;
info.dstTbUid = pSma->dstTbUid; info.dstTbUid = pSma->dstTbUid;
info.dstVgId = pSma->dstVgId; info.dstVgId = pSma->dstVgId;
SVgObj* pVg = mndAcquireVgroup(pMnode, pSma->dstVgId);
if (pVg == NULL) {
code = -1;
sdbRelease(pSdb, pSma);
return code;
}
info.epSet = mndGetVgroupEpset(pMnode, pVg);
info.expr = taosMemoryMalloc(pSma->exprLen + 1); info.expr = taosMemoryMalloc(pSma->exprLen + 1);
if (info.expr == NULL) { if (info.expr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
......
...@@ -36,12 +36,10 @@ target_sources( ...@@ -36,12 +36,10 @@ target_sources(
# tsdb # tsdb
"src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit.c"
# "src/tsdb/tsdbCommit2.c"
"src/tsdb/tsdbFile.c" "src/tsdb/tsdbFile.c"
"src/tsdb/tsdbFS.c" "src/tsdb/tsdbFS.c"
"src/tsdb/tsdbOpen.c" "src/tsdb/tsdbOpen.c"
"src/tsdb/tsdbMemTable.c" "src/tsdb/tsdbMemTable.c"
# "src/tsdb/tsdbMemTable2.c"
"src/tsdb/tsdbRead.c" "src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbReadImpl.c"
"src/tsdb/tsdbWrite.c" "src/tsdb/tsdbWrite.c"
......
...@@ -71,6 +71,9 @@ int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader); ...@@ -71,6 +71,9 @@ int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData); int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
// meta // meta
typedef struct SMeta SMeta; // todo: remove typedef struct SMeta SMeta; // todo: remove
typedef struct SMetaReader SMetaReader; typedef struct SMetaReader SMetaReader;
......
...@@ -58,28 +58,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, ST ...@@ -58,28 +58,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, ST
bool tsdbTbDataIterNext(STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter);
bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow); bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow);
int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
// tsdbMemTable2.c ==============================================================================================
// typedef struct SMemTable2 SMemTable2;
// typedef struct SMemData SMemData;
// typedef struct SMemDataIter SMemDataIter;
// int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable);
// void tsdbMemTableDestroy2(SMemTable2 *pMemTable);
// int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk);
// int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
// /* SMemDataIter */
// void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter);
// bool tsdbMemDataIterNext(SMemDataIter *pIter);
// void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow);
// // tsdbCommit2.c ==============================================================================================
// int32_t tsdbBegin2(STsdb *pTsdb);
// int32_t tsdbCommit2(STsdb *pTsdb);
// tsdbFile.c ============================================================================================== // tsdbFile.c ==============================================================================================
typedef int32_t TSDB_FILE_T; typedef int32_t TSDB_FILE_T;
typedef struct SDFInfo SDFInfo; typedef struct SDFInfo SDFInfo;
...@@ -700,17 +678,6 @@ typedef struct { ...@@ -700,17 +678,6 @@ typedef struct {
TSKEY eKey; TSKEY eKey;
} SDelInfo; } SDelInfo;
struct SMemTable2 {
STsdb *pTsdb;
int32_t nRef;
TSDBKEY minKey;
TSDBKEY maxKey;
int64_t nRows;
int64_t nDelOp;
SArray *aSkmInfo;
SArray *aMemData;
};
static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
TSDBKEY *pKey1 = (TSDBKEY *)p1; TSDBKEY *pKey1 = (TSDBKEY *)p1;
TSDBKEY *pKey2 = (TSDBKEY *)p2; TSDBKEY *pKey2 = (TSDBKEY *)p2;
...@@ -730,24 +697,6 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { ...@@ -730,24 +697,6 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
return 0; return 0;
} }
struct SMemData {
tb_uid_t suid;
tb_uid_t uid;
TSDBKEY minKey;
TSDBKEY maxKey;
SDelOp *delOpHead;
SDelOp *delOpTail;
SMemSkipList sl;
};
struct SMemDataIter {
STbData *pMemData;
int8_t backward;
TSDBROW *pRow;
SMemSkipListNode *pNode; // current node
TSDBROW row;
};
struct STbDataIter { struct STbDataIter {
STbData *pTbData; STbData *pTbData;
int8_t backward; int8_t backward;
......
...@@ -84,7 +84,6 @@ int32_t vnodeAsyncCommit(SVnode* pVnode); ...@@ -84,7 +84,6 @@ int32_t vnodeAsyncCommit(SVnode* pVnode);
int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
void vnodeSyncStart(SVnode* pVnode); void vnodeSyncStart(SVnode* pVnode);
void vnodeSyncClose(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode);
int32_t vnodeSyncAlter(SVnode* pVnode, SRpcMsg* pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -112,7 +112,7 @@ int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); ...@@ -112,7 +112,7 @@ int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
// tsdb // tsdb
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg);
int tsdbClose(STsdb** pTsdb); int tsdbClose(STsdb** pTsdb);
int tsdbBegin(STsdb* pTsdb); int32_t tsdbBegin(STsdb* pTsdb);
int32_t tsdbCommit(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
...@@ -161,18 +161,6 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore); ...@@ -161,18 +161,6 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
void tdUidStoreDestory(STbUidStore* pStore); void tdUidStoreDestory(STbUidStore* pStore);
void* tdUidStoreFree(STbUidStore* pStore); void* tdUidStoreFree(STbUidStore* pStore);
#if 0
int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
void tsdbUidStoreDestory(STbUidStore* pStore);
void* tsdbUidStoreFree(STbUidStore* pStore);
int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType);
#endif
typedef struct { typedef struct {
int8_t streamType; // sma or other int8_t streamType; // sma or other
int8_t dstType; int8_t dstType;
...@@ -232,8 +220,10 @@ struct SVnode { ...@@ -232,8 +220,10 @@ struct SVnode {
SWal* pWal; SWal* pWal;
STQ* pTq; STQ* pTq;
SSink* pSink; SSink* pSink;
int64_t sync;
tsem_t canCommit; tsem_t canCommit;
int64_t sync;
int32_t syncCount;
sem_t syncSem;
SQHandle* pQuery; SQHandle* pQuery;
}; };
......
...@@ -61,14 +61,14 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const ...@@ -61,14 +61,14 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
if (tTagToValArray((const STag *)data, &pTagVals) != 0) { if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
return -1; return -1;
} }
char key[512] = {0};
SIndexMultiTerm *terms = indexMultiTermCreate(); SIndexMultiTerm *terms = indexMultiTermCreate();
int16_t nCols = taosArrayGetSize(pTagVals); int16_t nCols = taosArrayGetSize(pTagVals);
for (int i = 0; i < nCols; i++) { for (int i = 0; i < nCols; i++) {
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type; char type = pTagVal->type;
sprintf(key, "%s_%s", tagName, pTagVal->pKey);
char * key = pTagVal->pKey;
int32_t nKey = strlen(key); int32_t nKey = strlen(key);
SIndexTerm *term = NULL; SIndexTerm *term = NULL;
...@@ -93,12 +93,11 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const ...@@ -93,12 +93,11 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
} else if (type == TSDB_DATA_TYPE_BOOL) { } else if (type == TSDB_DATA_TYPE_BOOL) {
int val = *(int *)(&pTagVal->i64); int val = *(int *)(&pTagVal->i64);
int len = 0; int len = 0;
term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len);
} }
if (term != NULL) { if (term != NULL) {
indexMultiTermAdd(terms, term); indexMultiTermAdd(terms, term);
} }
memset(key, 0, sizeof(key));
} }
tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid); tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
indexMultiTermDestroy(terms); indexMultiTermDestroy(terms);
......
...@@ -39,6 +39,9 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs ...@@ -39,6 +39,9 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) { static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) {
SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper); SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
if (pSW == NULL) {
return -1;
}
taosArrayPush(pRsp->blockSchema, &pSW); taosArrayPush(pRsp->blockSchema, &pSW);
return 0; return 0;
} }
......
...@@ -28,6 +28,8 @@ typedef struct { ...@@ -28,6 +28,8 @@ typedef struct {
int niters; // memory iterators int niters; // memory iterators
SCommitIter *iters; SCommitIter *iters;
bool isRFileSet; // read and commit FSET bool isRFileSet; // read and commit FSET
int32_t fid;
SDFileSet *pSet;
SReadH readh; SReadH readh;
SDFileSet wSet; SDFileSet wSet;
bool isDFileSame; bool isDFileSame;
...@@ -58,8 +60,12 @@ typedef struct { ...@@ -58,8 +60,12 @@ typedef struct {
#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows) #define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows)
#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
static void tsdbStartCommit(STsdb *pRepo); static int32_t tsdbCommitData(SCommitH *pCommith);
static void tsdbEndCommit(STsdb *pTsdb, int eno); static int32_t tsdbCommitDel(SCommitH *pCommith);
static int32_t tsdbCommitCache(SCommitH *pCommith);
static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitH *pCHandle);
static int32_t tsdbEndCommit(SCommitH *pCHandle, int eno);
static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo); static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo);
static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key);
static int tsdbNextCommitFid(SCommitH *pCommith); static int tsdbNextCommitFid(SCommitH *pCommith);
...@@ -67,7 +73,6 @@ static void tsdbDestroyCommitH(SCommitH *pCommith); ...@@ -67,7 +73,6 @@ static void tsdbDestroyCommitH(SCommitH *pCommith);
static int32_t tsdbCreateCommitIters(SCommitH *pCommith); static int32_t tsdbCreateCommitIters(SCommitH *pCommith);
static void tsdbDestroyCommitIters(SCommitH *pCommith); static void tsdbDestroyCommitIters(SCommitH *pCommith);
static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static void tsdbResetCommitFile(SCommitH *pCommith);
static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static int tsdbCommitToTable(SCommitH *pCommith, int tid); static int tsdbCommitToTable(SCommitH *pCommith, int tid);
static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx); static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx);
...@@ -88,8 +93,11 @@ static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *i ...@@ -88,8 +93,11 @@ static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *i
SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update); SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update);
static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn); static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn);
static int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup,
SMergeInfo *pMergeInfo);
int tsdbBegin(STsdb *pTsdb) { int32_t tsdbBegin(STsdb *pTsdb) {
if (!pTsdb) return 0; if (!pTsdb) return 0;
SMemTable *pMem; SMemTable *pMem;
...@@ -112,15 +120,50 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -112,15 +120,50 @@ int32_t tsdbCommit(STsdb *pTsdb) {
pTsdb->mem = NULL; pTsdb->mem = NULL;
// start commit // start commit
tsdbStartCommit(pTsdb); code = tsdbStartCommit(pTsdb, &commith);
if (tsdbInitCommitH(&commith, pTsdb) < 0) { if (code) {
return -1; goto _err;
}
// commit impl
code = tsdbCommitData(&commith);
if (code) {
goto _err;
} }
code = tsdbCommitDel(&commith);
if (code) {
goto _err;
}
code = tsdbCommitCache(&commith);
if (code) {
goto _err;
}
// end commit
code = tsdbEndCommit(&commith, 0);
if (code) {
goto _err;
}
return code;
_err:
tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
static int32_t tsdbCommitData(SCommitH *pCommith) {
int32_t fid;
SDFileSet *pSet = NULL;
int32_t code = 0;
STsdb *pTsdb = TSDB_COMMIT_REPO(pCommith);
// Skip expired memory data and expired FSET // Skip expired memory data and expired FSET
tsdbSeekCommitIter(&commith, commith.rtn.minKey); tsdbSeekCommitIter(pCommith, pCommith->rtn.minKey);
while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { while ((pSet = tsdbFSIterNext(&(pCommith->fsIter)))) {
if (pSet->fid < commith.rtn.minFid) { if (pSet->fid < pCommith->rtn.minFid) {
tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pTsdb), pSet->fid, tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pTsdb), pSet->fid,
TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
} else { } else {
...@@ -129,7 +172,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -129,7 +172,7 @@ int32_t tsdbCommit(STsdb *pTsdb) {
} }
// commit // commit
fid = tsdbNextCommitFid(&(commith)); fid = tsdbNextCommitFid(pCommith);
while (true) { while (true) {
// Loop over both on disk and memory // Loop over both on disk and memory
if (pSet == NULL && fid == TSDB_IVLD_FID) break; if (pSet == NULL && fid == TSDB_IVLD_FID) break;
...@@ -137,12 +180,12 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -137,12 +180,12 @@ int32_t tsdbCommit(STsdb *pTsdb) {
if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) {
// Only has existing FSET but no memory data to commit in this // Only has existing FSET but no memory data to commit in this
// existing FSET, only check if file in correct retention // existing FSET, only check if file in correct retention
if (tsdbApplyRtnOnFSet(pTsdb, pSet, &(commith.rtn)) < 0) { if (tsdbApplyRtnOnFSet(TSDB_COMMIT_REPO(pCommith), pSet, &(pCommith->rtn)) < 0) {
tsdbDestroyCommitH(&commith); tsdbDestroyCommitH(pCommith);
return -1; return -1;
} }
pSet = tsdbFSIterNext(&(commith.fsIter)); pSet = tsdbFSIterNext(&(pCommith->fsIter));
} else { } else {
// Has memory data to commit // Has memory data to commit
SDFileSet *pCSet; SDFileSet *pCSet;
...@@ -156,22 +199,30 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -156,22 +199,30 @@ int32_t tsdbCommit(STsdb *pTsdb) {
// Commit to an existing FSET // Commit to an existing FSET
pCSet = pSet; pCSet = pSet;
cfid = pSet->fid; cfid = pSet->fid;
pSet = tsdbFSIterNext(&(commith.fsIter)); pSet = tsdbFSIterNext(&(pCommith->fsIter));
} }
if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { if (tsdbCommitToFile(pCommith, pCSet, cfid) < 0) {
tsdbDestroyCommitH(&commith); tsdbDestroyCommitH(pCommith);
return -1; return -1;
} }
fid = tsdbNextCommitFid(&commith); fid = tsdbNextCommitFid(pCommith);
} }
} }
// end commit return code;
tsdbDestroyCommitH(&commith); }
tsdbEndCommit(pTsdb, TSDB_CODE_SUCCESS);
static int32_t tsdbCommitDel(SCommitH *pCommith) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitCache(SCommitH *pCommith) {
int32_t code = 0;
// TODO
return code; return code;
} }
...@@ -216,16 +267,6 @@ static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { ...@@ -216,16 +267,6 @@ static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
return 0; return 0;
} }
// int tsdbPrepareCommit(STsdb *pTsdb) {
// if (pTsdb->mem == NULL) return 0;
// ASSERT(pTsdb->imem == NULL);
// pTsdb->imem = pTsdb->mem;
// pTsdb->mem = NULL;
// return 0;
// }
void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
TSKEY minKey, midKey, maxKey, now; TSKEY minKey, midKey, maxKey, now;
...@@ -243,19 +284,32 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { ...@@ -243,19 +284,32 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
pRtn->minFid, pRtn->midFid, pRtn->maxFid); pRtn->minFid, pRtn->midFid, pRtn->maxFid);
} }
static void tsdbStartCommit(STsdb *pRepo) { static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitH *pCHandle) {
SMemTable *pMem = pRepo->imem; int32_t code = 0;
tsdbInfo("vgId:%d, start to commit", REPO_ID(pTsdb));
if (tsdbInitCommitH(pCHandle, pTsdb) < 0) {
return -1;
}
tsdbInfo("vgId:%d, start to commit", REPO_ID(pRepo)); tsdbStartFSTxn(pTsdb, 0, 0);
tsdbStartFSTxn(pRepo, 0, 0); return code;
} }
static void tsdbEndCommit(STsdb *pTsdb, int eno) { static int32_t tsdbEndCommit(SCommitH *pCHandle, int eno) {
int32_t code = 0;
STsdb *pTsdb = TSDB_COMMIT_REPO(pCHandle);
tsdbDestroyCommitH(pCHandle);
tsdbEndFSTxn(pTsdb); tsdbEndFSTxn(pTsdb);
tsdbMemTableDestroy(pTsdb->imem); tsdbMemTableDestroy(pTsdb->imem);
pTsdb->imem = NULL; pTsdb->imem = NULL;
tsdbInfo("vgId:%d, commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); tsdbInfo("vgId:%d, commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
return code;
} }
static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) { static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) {
...@@ -354,34 +408,73 @@ static void tsdbDestroyCommitH(SCommitH *pCommith) { ...@@ -354,34 +408,73 @@ static void tsdbDestroyCommitH(SCommitH *pCommith) {
tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith));
} }
static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { static int32_t tsdbCommitToFileStart(SCommitH *pCHandle, SDFileSet *pSet, int32_t fid) {
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); int32_t code = 0;
STsdb *pRepo = TSDB_COMMIT_REPO(pCHandle);
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
ASSERT(pSet == NULL || pSet->fid == fid); ASSERT(pSet == NULL || pSet->fid == fid);
tsdbResetCommitFile(pCommith); pCHandle->fid = fid;
tsdbGetFidKeyRange(pCfg->days, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); pCHandle->pSet = pSet;
pCHandle->isRFileSet = false;
pCHandle->isDFileSame = false;
pCHandle->isLFileSame = false;
taosArrayClear(pCHandle->aBlkIdx);
tsdbGetFidKeyRange(pCfg->days, pCfg->precision, fid, &(pCHandle->minKey), &(pCHandle->maxKey));
code = tsdbSetAndOpenCommitFile(pCHandle, pSet, fid);
// Set and open files return code;
if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { }
static int32_t tsdbCommitToFileImpl(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitToFileEnd(SCommitH *pCommith) {
int32_t code = 0;
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith);
if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) <
0) {
tsdbError("vgId:%d, failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), pCommith->fid,
tstrerror(terrno));
tsdbCloseCommitFile(pCommith, true);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pCommith->pSet);
return -1; return -1;
} }
#if 0
// Loop to commit each table data
for (int tid = 0; tid < pCommith->niters; tid++) {
SCommitIter *pIter = pCommith->iters + tid;
if (pIter->pTable == NULL) continue; if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) {
tsdbError("vgId:%d, failed to update FSET %d header since %s", REPO_ID(pRepo), pCommith->fid, tstrerror(terrno));
tsdbCloseCommitFile(pCommith, true);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pCommith->pSet);
return -1;
}
if (tsdbCommitToTable(pCommith, tid) < 0) { // Close commit file
tsdbCloseCommitFile(pCommith, true); tsdbCloseCommitFile(pCommith, false);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) {
return -1; return -1;
}
} }
#endif
return code;
}
static int32_t tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
int32_t code = 0;
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith);
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
// commit to file start
code = tsdbCommitToFileStart(pCommith, pSet, fid);
if (code) {
goto _err;
}
// Loop to commit each table data in mem and file // Loop to commit each table data in mem and file
int mIter = 0, fIter = 0; int mIter = 0, fIter = 0;
int nBlkIdx = taosArrayGetSize(pCommith->readh.aBlkIdx); int nBlkIdx = taosArrayGetSize(pCommith->readh.aBlkIdx);
...@@ -426,31 +519,16 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { ...@@ -426,31 +519,16 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
} }
} }
if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < // commit to file end
0) { code = tsdbCommitToFileEnd(pCommith);
tsdbError("vgId:%d, failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); if (code) {
tsdbCloseCommitFile(pCommith, true); goto _err;
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
return -1;
}
if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) {
tsdbError("vgId:%d, failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
tsdbCloseCommitFile(pCommith, true);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
return -1;
} }
// Close commit file return code;
tsdbCloseCommitFile(pCommith, false);
if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) {
return -1;
}
return 0; _err:
return code;
} }
static int32_t tsdbCreateCommitIters(SCommitH *pCommith) { static int32_t tsdbCreateCommitIters(SCommitH *pCommith) {
...@@ -507,13 +585,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) { ...@@ -507,13 +585,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) {
pCommith->niters = 0; pCommith->niters = 0;
} }
static void tsdbResetCommitFile(SCommitH *pCommith) {
pCommith->isRFileSet = false;
pCommith->isDFileSame = false;
pCommith->isLFileSame = false;
taosArrayClear(pCommith->aBlkIdx);
}
static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
SDiskID did; SDiskID did;
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdb *pRepo = TSDB_COMMIT_REPO(pCommith);
...@@ -1590,4 +1661,171 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p ...@@ -1590,4 +1661,171 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
} }
return false; return false;
}
static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
bool merge) {
if (pCols) {
if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
*ppSchema = tsdbGetTableSchemaImpl(pTsdb, pTable, false, false, TD_ROW_SVER(row));
if (*ppSchema == NULL) {
ASSERT(false);
return -1;
}
}
tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
}
return 0;
}
static int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup,
SMergeInfo *pMergeInfo) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
TSKEY rowKey = 0;
TSKEY fKey = 0;
// only fetch lastKey from mem data as file data not used in this function actually
TSKEY lastKey = TSKEY_INITIAL_VAL;
bool isRowDel = false;
int filterIter = 0;
STSRow *row = NULL;
SMergeInfo mInfo;
// TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by
// query handle)
if (pMergeInfo == NULL) pMergeInfo = &mInfo;
memset(pMergeInfo, 0, sizeof(*pMergeInfo));
pMergeInfo->keyFirst = INT64_MAX;
pMergeInfo->keyLast = INT64_MIN;
if (pCols) tdResetDataCols(pCols);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
// 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols
// 2. rowKey - would dup since Multi-Version supported
while (true) {
if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
if (fKey < rowKey) {
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
#if 1
} else if (fKey > rowKey) {
if (isRowDel) {
// TODO: support delete function
pMergeInfo->rowsDeleteFailed++;
} else {
if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
if (lastKey != rowKey) {
pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
if (pCols) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pCols->numOfRows;
}
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
}
lastKey = rowKey;
} else {
if (keepDup) {
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
} else {
// discard
}
}
}
tsdbTbDataIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
} else { // fkey == rowKey
if (isRowDel) { // TODO: support delete function(How to stands for delete in file? rowVersion = -1?)
ASSERT(!keepDup);
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsDeleteSucceed++;
pMergeInfo->nOperations++;
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
} else {
if (keepDup) {
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
if (lastKey != rowKey) {
pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
if (pCols) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pCols->numOfRows;
}
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
}
lastKey = rowKey;
} else {
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
}
} else {
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
}
}
tsdbTbDataIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
}
#endif
}
if (pCols && (lastKey != TSKEY_INITIAL_VAL)) {
++pCols->numOfRows;
}
return 0;
} }
\ No newline at end of file
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
typedef struct {
SMemTable2 *pMemTable;
int32_t minutes;
int8_t precision;
TSKEY nCommitKey;
int32_t fid;
TSKEY minKey;
TSKEY maxKey;
SReadH readh;
SDFileSet wSet;
SArray *aBlkIdx;
SArray *aSupBlk;
SArray *aSubBlk;
SArray *aDelInfo;
} SCommitH;
static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb);
static int32_t tsdbCommitEnd(SCommitH *pCHandle);
static int32_t tsdbCommitImpl(SCommitH *pCHandle);
int32_t tsdbBegin2(STsdb *pTsdb) {
int32_t code = 0;
ASSERT(pTsdb->mem == NULL);
code = tsdbMemTableCreate2(pTsdb, (SMemTable2 **)&pTsdb->mem);
if (code) {
tsdbError("vgId:%d failed to begin TSDB since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
goto _exit;
}
_exit:
return code;
}
int32_t tsdbCommit2(STsdb *pTsdb) {
int32_t code = 0;
SCommitH ch = {0};
// start to commit
code = tsdbCommitStart(&ch, pTsdb);
if (code) {
goto _exit;
}
// commit
code = tsdbCommitImpl(&ch);
if (code) {
goto _err;
}
// end commit
code = tsdbCommitEnd(&ch);
if (code) {
goto _exit;
}
_exit:
return code;
_err:
tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb) {
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem;
tsdbInfo("vgId:%d start to commit", TD_VID(pTsdb->pVnode));
// switch to commit
ASSERT(pTsdb->imem == NULL && pTsdb->mem);
pTsdb->imem = pTsdb->mem;
pTsdb->mem = NULL;
// open handle
pCHandle->pMemTable = pMemTable;
pCHandle->minutes = pTsdb->keepCfg.days;
pCHandle->precision = pTsdb->keepCfg.precision;
pCHandle->nCommitKey = pMemTable->minKey.ts;
code = tsdbInitReadH(&pCHandle->readh, pTsdb);
if (code) {
goto _err;
}
pCHandle->aBlkIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCHandle->aBlkIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pCHandle->aSupBlk = taosArrayInit(0, sizeof(SBlock));
if (pCHandle->aSupBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pCHandle->aSubBlk = taosArrayInit(0, sizeof(SBlock));
if (pCHandle->aSubBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pCHandle->aDelInfo = taosArrayInit(0, sizeof(SDelInfo));
if (pCHandle->aDelInfo == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
// start FS transaction
tsdbStartFSTxn(pTsdb, 0, 0);
return code;
_err:
return code;
}
static int32_t tsdbCommitEnd(SCommitH *pCHandle) {
int32_t code = 0;
STsdb *pTsdb = pCHandle->pMemTable->pTsdb;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->imem;
// end transaction
code = tsdbEndFSTxn(pTsdb);
if (code) {
goto _err;
}
// close handle
taosArrayClear(pCHandle->aDelInfo);
taosArrayClear(pCHandle->aSubBlk);
taosArrayClear(pCHandle->aSupBlk);
taosArrayClear(pCHandle->aBlkIdx);
tsdbDestroyReadH(&pCHandle->readh);
// destroy memtable (todo: unref it)
pTsdb->imem = NULL;
tsdbMemTableDestroy2(pMemTable);
tsdbInfo("vgId:%d commit over", TD_VID(pTsdb->pVnode));
return code;
_err:
return code;
}
static int32_t tsdbCommitTableStart(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitTableEnd(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitTable(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) {
int32_t code = 0;
SMemDataIter iter = {0};
// commit table start
code = tsdbCommitTableStart(pCHandle);
if (code) {
goto _err;
}
// commit table impl
if (pMemData && pBlockIdx) {
// TODO
} else if (pMemData) {
// TODO
} else {
// TODO
}
// commit table end
code = tsdbCommitTableEnd(pCHandle);
if (code) {
goto _err;
}
return code;
_err:
return code;
}
static int32_t tsdbTableIdCmprFn(const void *p1, const void *p2) {
TABLEID *pId1 = (TABLEID *)p1;
TABLEID *pId2 = (TABLEID *)p2;
if (pId1->suid < pId2->suid) {
return -1;
} else if (pId1->suid > pId2->suid) {
return 1;
}
if (pId1->uid < pId2->uid) {
return -1;
} else if (pId1->uid > pId2->uid) {
return 1;
}
return 0;
}
static int32_t tsdbWriteBlockIdx(SDFile *pFile, SArray *pArray, uint8_t **ppBuf) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitFileStart(SCommitH *pCHandle) {
int32_t code = 0;
STsdb *pTsdb = pCHandle->pMemTable->pTsdb;
SDFileSet *pSet = NULL;
taosArrayClear(pCHandle->aBlkIdx);
return code;
}
static int32_t tsdbCommitFileEnd(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitFile(SCommitH *pCHandle) {
int32_t code = 0;
SMemData *pMemData;
SBlockIdx *pBlockIdx;
int32_t iMemData;
int32_t nMemData;
int32_t iBlockIdx;
int32_t nBlockIdx;
// commit file start
code = tsdbCommitFileStart(pCHandle);
if (code) {
goto _err;
}
// commit file impl
iMemData = 0;
nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData);
iBlockIdx = 0;
nBlockIdx = 0; // todo
for (;;) {
if (iMemData >= nMemData && iBlockIdx >= nBlockIdx) break;
pMemData = NULL;
pBlockIdx = NULL;
if (iMemData < nMemData) {
pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData);
}
if (iBlockIdx < nBlockIdx) {
// pBlockIdx = ;
}
if (pMemData && pBlockIdx) {
int32_t c = tsdbTableIdCmprFn(pMemData, pBlockIdx);
if (c < 0) {
iMemData++;
pBlockIdx = NULL;
} else if (c == 0) {
iMemData++;
iBlockIdx++;
} else {
iBlockIdx++;
pMemData = NULL;
}
} else {
if (pMemData) {
iMemData++;
} else {
iBlockIdx++;
}
}
code = tsdbCommitTable(pCHandle, pMemData, pBlockIdx);
if (code) {
goto _err;
}
}
// commit file end
code = tsdbCommitFileEnd(pCHandle);
if (code) {
goto _err;
}
return code;
_err:
return code;
}
static int32_t tsdbCommitData(SCommitH *pCHandle) {
int32_t code = 0;
int32_t fid;
if (pCHandle->pMemTable->nRows == 0) goto _exit;
// loop to commit to each file
for (;;) {
if (pCHandle->nCommitKey == TSKEY_MAX) break;
pCHandle->fid = TSDB_KEY_FID(pCHandle->nCommitKey, pCHandle->minutes, pCHandle->precision);
tsdbGetFidKeyRange(pCHandle->minutes, pCHandle->precision, pCHandle->fid, &pCHandle->minKey, &pCHandle->maxKey);
code = tsdbCommitFile(pCHandle);
if (code) {
goto _err;
}
}
_exit:
return code;
_err:
return code;
}
static int32_t delInfoCmprFn(const void *p1, const void *p2) {
SDelInfo *pDelInfo1 = (SDelInfo *)p1;
SDelInfo *pDelInfo2 = (SDelInfo *)p2;
if (pDelInfo1->suid < pDelInfo2->suid) {
return -1;
} else if (pDelInfo1->suid > pDelInfo2->suid) {
return 1;
}
if (pDelInfo1->uid < pDelInfo2->uid) {
return -1;
} else if (pDelInfo1->uid > pDelInfo2->uid) {
return 1;
}
if (pDelInfo1->version < pDelInfo2->version) {
return -1;
} else if (pDelInfo1->version > pDelInfo2->version) {
return 1;
}
return 0;
}
static int32_t tsdbCommitDelete(SCommitH *pCHandle) {
int32_t code = 0;
SDelInfo delInfo;
SMemData *pMemData;
if (pCHandle->pMemTable->nDelOp == 0) goto _exit;
// load del array (todo)
// loop to append SDelInfo
for (int32_t iMemData = 0; iMemData < taosArrayGetSize(pCHandle->pMemTable->aMemData); iMemData++) {
pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData);
for (SDelOp *pDelOp = pMemData->delOpHead; pDelOp; pDelOp = pDelOp->pNext) {
delInfo = (SDelInfo){.suid = pMemData->suid,
.uid = pMemData->uid,
.version = pDelOp->version,
.sKey = pDelOp->sKey,
.eKey = pDelOp->eKey};
if (taosArrayPush(pCHandle->aDelInfo, &delInfo) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
}
}
taosArraySort(pCHandle->aDelInfo, delInfoCmprFn);
// write to new file
_exit:
return code;
_err:
return code;
}
static int32_t tsdbCommitCache(SCommitH *pCHandle) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbCommitImpl(SCommitH *pCHandle) {
int32_t code = 0;
// commit data
code = tsdbCommitData(pCHandle);
if (code) {
goto _err;
}
// commit delete
code = tsdbCommitDelete(pCHandle);
if (code) {
goto _err;
}
// commit cache if need (todo)
if (0) {
code = tsdbCommitCache(pCHandle);
if (code) {
goto _err;
}
}
return code;
_err:
return code;
}
\ No newline at end of file
...@@ -188,23 +188,6 @@ _err: ...@@ -188,23 +188,6 @@ _err:
return code; return code;
} }
static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
bool merge) {
if (pCols) {
if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
*ppSchema = tsdbGetTableSchemaImpl(pTsdb, pTable, false, false, TD_ROW_SVER(row));
if (*ppSchema == NULL) {
ASSERT(false);
return -1;
}
}
tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
}
return 0;
}
int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter) { int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter) {
int32_t code = 0; int32_t code = 0;
...@@ -310,166 +293,6 @@ bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow) { ...@@ -310,166 +293,6 @@ bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow) {
return true; return true;
} }
/**
* This is an important function to load data or try to load data from memory skiplist iterator.
*
* This function load memory data until:
* 1. iterator ends
* 2. data key exceeds maxKey
* 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
* 4. operations in pCols not exceeds its max capacity if pCols is given
*
* The function tries to procceed AS MUCH AS POSSIBLE.
*/
int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
TSKEY rowKey = 0;
TSKEY fKey = 0;
// only fetch lastKey from mem data as file data not used in this function actually
TSKEY lastKey = TSKEY_INITIAL_VAL;
bool isRowDel = false;
int filterIter = 0;
STSRow *row = NULL;
SMergeInfo mInfo;
// TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by
// query handle)
if (pMergeInfo == NULL) pMergeInfo = &mInfo;
memset(pMergeInfo, 0, sizeof(*pMergeInfo));
pMergeInfo->keyFirst = INT64_MAX;
pMergeInfo->keyLast = INT64_MIN;
if (pCols) tdResetDataCols(pCols);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
// 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols
// 2. rowKey - would dup since Multi-Version supported
while (true) {
if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
if (fKey < rowKey) {
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
#if 1
} else if (fKey > rowKey) {
if (isRowDel) {
// TODO: support delete function
pMergeInfo->rowsDeleteFailed++;
} else {
if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
if (lastKey != rowKey) {
pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
if (pCols) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pCols->numOfRows;
}
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
}
lastKey = rowKey;
} else {
if (keepDup) {
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
} else {
// discard
}
}
}
tsdbTbDataIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
} else { // fkey == rowKey
if (isRowDel) { // TODO: support delete function(How to stands for delete in file? rowVersion = -1?)
ASSERT(!keepDup);
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsDeleteSucceed++;
pMergeInfo->nOperations++;
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
} else {
if (keepDup) {
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
if (lastKey != rowKey) {
pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
if (pCols) {
if (lastKey != TSKEY_INITIAL_VAL) {
++pCols->numOfRows;
}
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
}
lastKey = rowKey;
} else {
tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
}
} else {
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
}
}
tsdbTbDataIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || TD_ROW_KEY(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = TD_ROW_KEY(row);
isRowDel = TD_ROW_IS_DELETED(row);
}
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
}
#endif
}
if (pCols && (lastKey != TSKEY_INITIAL_VAL)) {
++pCols->numOfRows;
}
return 0;
}
static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) { static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
int32_t code = 0; int32_t code = 0;
int32_t idx = 0; int32_t idx = 0;
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
typedef struct {
tb_uid_t uid;
STSchema *pTSchema;
} SSkmInfo;
#define SL_MAX_LEVEL 5
#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
#define SL_NODE_FORWARD(n, l) ((n)->forwards[l])
#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level))
#define SL_MOVE_BACKWARD 0x1
#define SL_MOVE_FROM_POS 0x2
static int32_t tsdbGetOrCreateMemData(SMemTable2 *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData);
static int memDataPCmprFn(const void *p1, const void *p2);
static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl);
static int32_t tsdbInsertTableDataImpl(SMemTable2 *pMemTable, SMemData *pMemData, int64_t version,
SVSubmitBlk *pSubmitBlk);
static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags);
// SMemTable ==============================================
int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable) {
int32_t code = 0;
SMemTable2 *pMemTable = NULL;
pMemTable = (SMemTable2 *)taosMemoryCalloc(1, sizeof(*pMemTable));
if (pMemTable == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pMemTable->pTsdb = pTsdb;
pMemTable->nRef = 1;
pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
pMemTable->nRows = 0;
pMemTable->nDelOp = 0;
pMemTable->aMemData = taosArrayInit(512, sizeof(SMemData *));
if (pMemTable->aMemData == NULL) {
taosMemoryFree(pMemTable);
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*ppMemTable = pMemTable;
return code;
_err:
*ppMemTable = NULL;
return code;
}
void tsdbMemTableDestroy2(SMemTable2 *pMemTable) {
taosArrayDestroyEx(pMemTable->aMemData, NULL /*TODO*/);
taosMemoryFree(pMemTable);
}
int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) {
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem; // TODO
SMemData *pMemData;
TSDBROW row = {.version = version};
ASSERT(pMemTable);
ASSERT(pSubmitBlk->nData > 0);
{
// check if table exists (todo)
}
code = tsdbGetOrCreateMemData(pMemTable, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData);
if (code) {
tsdbError("vgId:%d, failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
goto _err;
}
// do insert
code = tsdbInsertTableDataImpl(pMemTable, pMemData, version, pSubmitBlk);
if (code) {
goto _err;
}
return code;
_err:
return code;
}
int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) {
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem; // TODO
SMemData *pMemData;
SVBufPool *pPool = pTsdb->pVnode->inUse;
ASSERT(pMemTable);
{
// check if table exists (todo)
}
code = tsdbGetOrCreateMemData(pMemTable, suid, uid, &pMemData);
if (code) {
goto _err;
}
// do delete
SDelOp *pDelOp = (SDelOp *)vnodeBufPoolMalloc(pPool, sizeof(*pDelOp));
if (pDelOp == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pDelOp->version = version;
pDelOp->sKey = sKey;
pDelOp->eKey = eKey;
pDelOp->pNext = NULL;
if (pMemData->delOpHead == NULL) {
ASSERT(pMemData->delOpTail == NULL);
pMemData->delOpHead = pMemData->delOpTail = pDelOp;
} else {
pMemData->delOpTail->pNext = pDelOp;
pMemData->delOpTail = pDelOp;
}
{
// update the state of pMemTable, pMemData, last and lastrow (todo)
}
pMemTable->nDelOp++;
tsdbDebug("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
" since %s",
TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
return code;
_err:
tsdbError("vgId:%d, failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
" since %s",
TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
return code;
}
void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter) {
SMemSkipListNode *pos[SL_MAX_LEVEL];
pIter->pMemData = pMemData;
pIter->backward = backward;
pIter->pRow = NULL;
if (pKey == NULL) {
// create from head or tail
if (backward) {
pIter->pNode = SL_NODE_BACKWARD(pMemData->sl.pTail, 0);
} else {
pIter->pNode = SL_NODE_FORWARD(pMemData->sl.pHead, 0);
}
} else {
// create from a key
if (backward) {
memDataMovePosTo(pMemData, pos, pKey, SL_MOVE_BACKWARD);
pIter->pNode = SL_NODE_BACKWARD(pos[0], 0);
} else {
memDataMovePosTo(pMemData, pos, pKey, 0);
pIter->pNode = SL_NODE_FORWARD(pos[0], 0);
}
}
}
bool tsdbMemDataIterNext(SMemDataIter *pIter) {
SMemSkipListNode *pHead = pIter->pMemData->sl.pHead;
SMemSkipListNode *pTail = pIter->pMemData->sl.pTail;
pIter->pRow = NULL;
if (pIter->backward) {
ASSERT(pIter->pNode != pTail);
if (pIter->pNode == pHead) {
return false;
}
pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0);
if (pIter->pNode == pHead) {
return false;
}
} else {
ASSERT(pIter->pNode != pHead);
if (pIter->pNode == pTail) {
return false;
}
pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0);
if (pIter->pNode == pTail) {
return false;
}
}
return true;
}
void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow) {
if (pIter->pRow) {
*ppRow = pIter->pRow;
} else {
SMemSkipListNode *pHead = pIter->pMemData->sl.pHead;
SMemSkipListNode *pTail = pIter->pMemData->sl.pTail;
if (pIter->backward) {
ASSERT(pIter->pNode != pTail);
if (pIter->pNode == pHead) {
*ppRow = NULL;
} else {
tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
*ppRow = &pIter->row;
}
} else {
ASSERT(pIter->pNode != pHead);
if (pIter->pNode == pTail) {
*ppRow = NULL;
} else {
tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
*ppRow = &pIter->row;
}
}
}
}
static int32_t tsdbGetOrCreateMemData(SMemTable2 *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) {
int32_t code = 0;
int32_t idx = 0;
SMemData *pMemDataT = &(SMemData){.suid = suid, .uid = uid};
SMemData *pMemData = NULL;
SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
// get
idx = taosArraySearchIdx(pMemTable->aMemData, &pMemDataT, memDataPCmprFn, TD_GE);
if (idx >= 0) {
pMemData = (SMemData *)taosArrayGet(pMemTable->aMemData, idx);
if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit;
}
// create
pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData) + SL_NODE_SIZE(maxLevel) * 2);
if (pMemData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pMemData->suid = suid;
pMemData->uid = uid;
pMemData->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
pMemData->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
pMemData->delOpHead = pMemData->delOpTail = NULL;
pMemData->sl.seed = taosRand();
pMemData->sl.size = 0;
pMemData->sl.maxLevel = maxLevel;
pMemData->sl.level = 0;
pMemData->sl.pHead = (SMemSkipListNode *)&pMemData[1];
pMemData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pMemData->sl.pHead, SL_NODE_SIZE(maxLevel));
pMemData->sl.pHead->level = maxLevel;
pMemData->sl.pTail->level = maxLevel;
for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) {
SL_NODE_FORWARD(pMemData->sl.pHead, iLevel) = pMemData->sl.pTail;
SL_NODE_BACKWARD(pMemData->sl.pHead, iLevel) = NULL;
SL_NODE_BACKWARD(pMemData->sl.pTail, iLevel) = pMemData->sl.pHead;
SL_NODE_FORWARD(pMemData->sl.pTail, iLevel) = NULL;
}
if (idx < 0) idx = 0;
if (taosArrayInsert(pMemTable->aMemData, idx, &pMemData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
_exit:
*ppMemData = pMemData;
return code;
_err:
*ppMemData = NULL;
return code;
}
static int memDataPCmprFn(const void *p1, const void *p2) {
SMemData *pMemData1 = *(SMemData **)p1;
SMemData *pMemData2 = *(SMemData **)p2;
if (pMemData1->suid < pMemData2->suid) {
return -1;
} else if (pMemData1->suid > pMemData2->suid) {
return 1;
}
if (pMemData1->uid < pMemData2->uid) {
return -1;
} else if (pMemData1->uid > pMemData2->uid) {
return 1;
}
return 0;
}
static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) {
int32_t n = 0;
n += tPutI64(p ? p + n : p, pRow->version);
n += tPutTSRow(p ? p + n : p, &pRow->tsRow);
return n;
}
static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) {
int32_t n = 0;
n += tGetI64(p + n, &pRow->version);
n += tGetTSRow(p + n, &pRow->tsRow);
return n;
}
static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
int8_t level = 1;
int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
const uint32_t factor = 4;
while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
level++;
}
return level;
}
static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags) {
SMemSkipListNode *px;
SMemSkipListNode *pn;
TSDBKEY *pTKey;
int c;
int backward = flags & SL_MOVE_BACKWARD;
int fromPos = flags & SL_MOVE_FROM_POS;
if (backward) {
px = pMemData->sl.pTail;
for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) {
pos[iLevel] = px;
}
if (pMemData->sl.level) {
if (fromPos) px = pos[pMemData->sl.level - 1];
for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) {
pn = SL_NODE_BACKWARD(px, iLevel);
while (pn != pMemData->sl.pHead) {
pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
c = tsdbKeyCmprFn(pTKey, pKey);
if (c <= 0) {
break;
} else {
px = pn;
pn = SL_NODE_BACKWARD(px, iLevel);
}
}
pos[iLevel] = px;
}
}
} else {
px = pMemData->sl.pHead;
for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) {
pos[iLevel] = px;
}
if (pMemData->sl.level) {
if (fromPos) px = pos[pMemData->sl.level - 1];
for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) {
pn = SL_NODE_FORWARD(px, iLevel);
while (pn != pMemData->sl.pHead) {
pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
c = tsdbKeyCmprFn(pTKey, pKey);
if (c >= 0) {
break;
} else {
px = pn;
pn = SL_NODE_FORWARD(px, iLevel);
}
}
pos[iLevel] = px;
}
}
}
}
static int32_t memDataDoPut(SMemTable2 *pMemTable, SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow,
int8_t forward) {
int32_t code = 0;
int8_t level;
SMemSkipListNode *pNode;
SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
// node
level = tsdbMemSkipListRandLevel(&pMemData->sl);
pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow));
if (pNode == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pNode->level = level;
for (int8_t iLevel = 0; iLevel < level; iLevel++) {
SL_NODE_FORWARD(pNode, iLevel) = NULL;
SL_NODE_BACKWARD(pNode, iLevel) = NULL;
}
tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow);
// put
for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) {
SMemSkipListNode *px = pos[iLevel];
if (forward) {
SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel);
SL_NODE_FORWARD(pNode, iLevel) = pNext;
SL_NODE_BACKWARD(pNode, iLevel) = px;
SL_NODE_BACKWARD(pNext, iLevel) = pNode;
SL_NODE_FORWARD(px, iLevel) = pNode;
} else {
SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel);
SL_NODE_FORWARD(pNode, iLevel) = px;
SL_NODE_BACKWARD(pNode, iLevel) = pPrev;
SL_NODE_FORWARD(pPrev, iLevel) = pNode;
SL_NODE_BACKWARD(px, iLevel) = pNode;
}
}
pMemData->sl.size++;
if (pMemData->sl.level < pNode->level) {
pMemData->sl.level = pNode->level;
}
_exit:
return code;
}
static int32_t tsdbInsertTableDataImpl(SMemTable2 *pMemTable, SMemData *pMemData, int64_t version,
SVSubmitBlk *pSubmitBlk) {
int32_t code = 0;
int32_t n = 0;
uint8_t *p = pSubmitBlk->pData;
int32_t nRow = 0;
TSDBROW row = {.version = version};
SMemSkipListNode *pos[SL_MAX_LEVEL];
ASSERT(pSubmitBlk->nData);
// backward put first data
n += tGetTSRow(p + n, &row.tsRow);
ASSERT(n <= pSubmitBlk->nData);
memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_BACKWARD);
code = memDataDoPut(pMemTable, pMemData, pos, &row, 0);
if (code) {
goto _exit;
}
nRow++;
if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->minKey) < 0) {
pMemData->minKey = *(TSDBKEY *)&row;
}
if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->minKey) < 0) {
pMemTable->minKey = *(TSDBKEY *)&row;
}
// forward put rest
for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) {
pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel);
}
while (n < pSubmitBlk->nData) {
n += tGetTSRow(p + n, &row.tsRow);
ASSERT(n <= pSubmitBlk->nData);
memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_FROM_POS);
code = memDataDoPut(pMemTable, pMemData, pos, &row, 1);
if (code) {
goto _exit;
}
nRow++;
}
if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->maxKey) > 0) {
pMemData->maxKey = *(TSDBKEY *)&row;
}
if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->maxKey) > 0) {
pMemTable->maxKey = *(TSDBKEY *)&row;
}
pMemTable->nRows += nRow;
_exit:
return code;
}
...@@ -196,33 +196,6 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) { ...@@ -196,33 +196,6 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
} else { } else {
pReadh->pBlkIdx = (SBlockIdx *)p; pReadh->pBlkIdx = (SBlockIdx *)p;
} }
// size_t size = taosArrayGetSize(pReadh->aBlkIdx);
// if (size > 0) {
// while (true) {
// if (pReadh->cidx >= size) {
// pReadh->pBlkIdx = NULL;
// break;
// }
// SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
// if (pBlkIdx->uid == TABLE_TID(pTable)) {
// if (pBlkIdx->uid == TABLE_UID(pTable)) {
// pReadh->pBlkIdx = pBlkIdx;
// } else {
// pReadh->pBlkIdx = NULL;
// }
// pReadh->cidx++;
// break;
// } else if (pBlkIdx->uid > TABLE_TID(pTable)) {
// pReadh->pBlkIdx = NULL;
// break;
// } else {
// pReadh->cidx++;
// }
// }
// } else {
// pReadh->pBlkIdx = NULL;
// }
return 0; return 0;
} }
......
...@@ -81,7 +81,9 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { ...@@ -81,7 +81,9 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
pVnode->state.applied = info.state.committed; pVnode->state.applied = info.state.committed;
pVnode->pTfs = pTfs; pVnode->pTfs = pTfs;
pVnode->msgCb = msgCb; pVnode->msgCb = msgCb;
pVnode->syncCount = 0;
tsem_init(&pVnode->syncSem, 0, 0);
tsem_init(&(pVnode->canCommit), 0, 1); tsem_init(&(pVnode->canCommit), 0, 1);
// open buffer pool // open buffer pool
...@@ -175,6 +177,7 @@ void vnodeClose(SVnode *pVnode) { ...@@ -175,6 +177,7 @@ void vnodeClose(SVnode *pVnode) {
vnodeCloseBufPool(pVnode); vnodeCloseBufPool(pVnode);
// destroy handle // destroy handle
tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&(pVnode->canCommit));
tsem_destroy(&pVnode->syncSem);
taosMemoryFree(pVnode); taosMemoryFree(pVnode);
} }
} }
......
...@@ -91,9 +91,6 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { ...@@ -91,9 +91,6 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
} }
} break; } break;
case TDMT_VND_ALTER_REPLICA: {
code = vnodeSyncAlter(pVnode, pMsg);
} break;
default: default:
break; break;
} }
...@@ -107,7 +104,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp ...@@ -107,7 +104,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
int32_t len; int32_t len;
int32_t ret; int32_t ret;
vTrace("vgId:%d, start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), vTrace("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
version); version);
pVnode->state.applied = version; pVnode->state.applied = version;
...@@ -173,7 +170,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp ...@@ -173,7 +170,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
break; break;
} }
vTrace("vgId:%d, process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); vTrace("vgId:%d, process %s request success, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
......
...@@ -16,41 +16,41 @@ ...@@ -16,41 +16,41 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "vnd.h" #include "vnd.h"
static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg); static inline bool vnodeIsMsgBlock(tmsg_t type) {
static int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg); return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA);
static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode); }
static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
SSyncInfo syncInfo = {
.vgId = pVnode->config.vgId,
.isStandBy = pVnode->config.standby,
.syncCfg = pVnode->config.syncCfg,
.pWal = pVnode->pWal,
.msgcb = NULL,
.FpSendMsg = vnodeSyncSendMsg,
.FpEqMsg = vnodeSyncEqMsg,
};
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", path, TD_DIRSEP); static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) {
syncInfo.pFsm = vnodeSyncMakeFsm(pVnode); if (!vnodeIsMsgBlock(type)) return;
pVnode->sync = syncOpen(&syncInfo); int32_t count = atomic_add_fetch_32(&pVnode->syncCount, 1);
if (pVnode->sync <= 0) { vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr()); }
return -1;
}
setPingTimerMS(pVnode->sync, 3000); static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
setElectTimerMS(pVnode->sync, 500); int32_t count = atomic_load_32(&pVnode->syncCount);
setHeartbeatTimerMS(pVnode->sync, 100); if (count <= 0) return;
return 0;
vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
tsem_wait(&pVnode->syncSem);
}
static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
if (!vnodeIsMsgBlock(type)) return;
int32_t count = atomic_load_32(&pVnode->syncCount);
if (count <= 0) return;
count = atomic_sub_fetch_32(&pVnode->syncCount, 1);
vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
if (count <= 0) {
tsem_post(&pVnode->syncSem);
}
} }
int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) { static int32_t vnodeProcessSyncReconfigReq(SVnode *pVnode, SRpcMsg *pMsg) {
SAlterVnodeReq req = {0}; SAlterVnodeReq req = {0};
if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) { if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
...@@ -66,29 +66,98 @@ int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) { ...@@ -66,29 +66,98 @@ int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) {
vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort); vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort);
} }
int32_t code = syncReconfig(pVnode->sync, &cfg); return syncReconfig(pVnode->sync, &cfg);
if (code == TAOS_SYNC_PROPOSE_SUCCESS) { }
// todo refactor
SRpcMsg rsp = {.info = pMsg->info, .code = 0}; void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
tmsgSendRsp(&rsp); SVnode *pVnode = pInfo->ahandle;
return TSDB_CODE_ACTION_IN_PROGRESS; int32_t vgId = pVnode->config.vgId;
int32_t code = 0;
SRpcMsg *pMsg = NULL;
for (int32_t m = 0; m < numOfMsgs; m++) {
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
vTrace("vgId:%d, msg:%p get from vnode-write queue handle:%p", vgId, pMsg, pMsg->info.handle);
if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
code = vnodeProcessSyncReconfigReq(pVnode, pMsg);
} else {
code = vnodePreprocessReq(pVnode, pMsg);
if (code != 0) {
vError("vgId:%d, failed to pre-process msg:%p since %s", vgId, pMsg, terrstr());
} else {
code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType));
}
}
if (code == 0) {
vnodeAccumBlockMsg(pVnode, pMsg->msgType);
if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
// todo refactor
SRpcMsg rsp = {.code = code, .info = pMsg->info};
tmsgSendRsp(&rsp);
}
} else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
SEpSet newEpSet = {0};
syncGetEpSet(pVnode->sync, &newEpSet);
SEp *pEp = &newEpSet.eps[newEpSet.inUse];
if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
}
vTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps,
newEpSet.inUse);
for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
vTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", vgId, pMsg, i, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
}
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
tmsgSendRedirectRsp(&rsp, &newEpSet);
} else {
if (terrno != 0) code = terrno;
vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code);
SRpcMsg rsp = {.code = code, .info = pMsg->info};
tmsgSendRsp(&rsp);
}
vTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
} }
return code; vnodeWaitBlockMsg(pVnode);
} }
void vnodeSyncStart(SVnode *pVnode) { void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
syncSetMsgCb(pVnode->sync, &pVnode->msgCb); SVnode *pVnode = pInfo->ahandle;
if (pVnode->config.standby) { int32_t vgId = pVnode->config.vgId;
syncStartStandBy(pVnode->sync); int32_t code = 0;
} else { SRpcMsg *pMsg = NULL;
syncStart(pVnode->sync);
for (int32_t i = 0; i < numOfMsgs; ++i) {
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
vTrace("vgId:%d, msg:%p get from vnode-apply queue, handle:%p", vgId, pMsg, pMsg->info.handle);
SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info};
if (rsp.code == 0) {
if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->conn.applyIndex, &rsp) < 0) {
rsp.code = terrno;
vError("vgId:%d, msg:%p failed to apply since %s", vgId, pMsg, terrstr());
}
}
vnodePostBlockMsg(pVnode, pMsg->msgType);
if (rsp.info.handle != NULL) {
tmsgSendRsp(&rsp);
}
vTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, rsp.code);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
} }
} }
void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
if (code != 0) { if (code != 0) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
...@@ -97,7 +166,7 @@ int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { ...@@ -97,7 +166,7 @@ int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
return code; return code;
} }
int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { static int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = tmsgSendReq(pEpSet, pMsg); int32_t code = tmsgSendReq(pEpSet, pMsg);
if (code != 0) { if (code != 0) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
...@@ -106,53 +175,43 @@ int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { ...@@ -106,53 +175,43 @@ int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
return code; return code;
} }
int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) {
vnodeGetSnapshot(pFsm->data, pSnapshot); vnodeGetSnapshot(pFsm->data, pSnapshot);
return 0; return 0;
} }
void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { static void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode)); vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode));
// todo rpc response here // todo rpc response here
// build rpc msg
// put into apply queue
} }
void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
SVnode *pVnode = pFsm->data;
SSnapshot snapshot = {0};
SyncIndex beginIndex = SYNC_INDEX_INVALID; SyncIndex beginIndex = SYNC_INDEX_INVALID;
char logBuf[256] = {0};
if (pFsm->FpGetSnapshot != NULL) { if (pFsm->FpGetSnapshot != NULL) {
SSnapshot snapshot = {0}; (*pFsm->FpGetSnapshot)(pFsm, &snapshot);
pFsm->FpGetSnapshot(pFsm, &snapshot);
beginIndex = snapshot.lastApplyIndex; beginIndex = snapshot.lastApplyIndex;
} }
if (cbMeta.index > beginIndex) { if (cbMeta.index > beginIndex) {
char logBuf[256] = {0};
snprintf( snprintf(
logBuf, sizeof(logBuf), logBuf, sizeof(logBuf),
"==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n", "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
SVnode *pVnode = pFsm->data; SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index};
SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta); rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
SRpcMsg applyMsg; memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg); syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info);
syncApplyMsgDestroy(pSyncApplyMsg); tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
// recover handle for response
SRpcMsg saveRpcMsg;
int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg);
if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) {
applyMsg.info = saveRpcMsg.info;
} else {
applyMsg.info.handle = NULL;
applyMsg.info.ahandle = NULL;
}
// put to applyQ
tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg);
} else { } else {
char logBuf[256] = {0}; char logBuf[256] = {0};
snprintf(logBuf, sizeof(logBuf), snprintf(logBuf, sizeof(logBuf),
...@@ -164,7 +223,7 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) ...@@ -164,7 +223,7 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta)
} }
} }
void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
char logBuf[256] = {0}; char logBuf[256] = {0};
snprintf(logBuf, sizeof(logBuf), snprintf(logBuf, sizeof(logBuf),
"==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
...@@ -172,14 +231,14 @@ void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMet ...@@ -172,14 +231,14 @@ void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMet
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
} }
void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
char logBuf[256] = {0}; char logBuf[256] = {0};
snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
} }
SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
pFsm->data = pVnode; pFsm->data = pVnode;
pFsm->FpCommitCb = vnodeSyncCommitMsg; pFsm->FpCommitCb = vnodeSyncCommitMsg;
...@@ -188,6 +247,42 @@ SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { ...@@ -188,6 +247,42 @@ SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
pFsm->FpGetSnapshot = vnodeSyncGetSnapshot; pFsm->FpGetSnapshot = vnodeSyncGetSnapshot;
pFsm->FpRestoreFinishCb = NULL; pFsm->FpRestoreFinishCb = NULL;
pFsm->FpReConfigCb = vnodeSyncReconfig; pFsm->FpReConfigCb = vnodeSyncReconfig;
return pFsm; return pFsm;
} }
\ No newline at end of file
int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
SSyncInfo syncInfo = {
.vgId = pVnode->config.vgId,
.isStandBy = pVnode->config.standby,
.syncCfg = pVnode->config.syncCfg,
.pWal = pVnode->pWal,
.msgcb = NULL,
.FpSendMsg = vnodeSyncSendMsg,
.FpEqMsg = vnodeSyncEqMsg,
};
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", path, TD_DIRSEP);
syncInfo.pFsm = vnodeSyncMakeFsm(pVnode);
pVnode->sync = syncOpen(&syncInfo);
if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
return -1;
}
setPingTimerMS(pVnode->sync, 3000);
setElectTimerMS(pVnode->sync, 500);
setHeartbeatTimerMS(pVnode->sync, 100);
return 0;
}
void vnodeSyncStart(SVnode *pVnode) {
syncSetMsgCb(pVnode->sync, &pVnode->msgCb);
if (pVnode->config.standby) {
syncStartStandBy(pVnode->sync);
} else {
syncStart(pVnode->sync);
}
}
void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); }
...@@ -173,7 +173,6 @@ typedef struct SCtgJob { ...@@ -173,7 +173,6 @@ typedef struct SCtgJob {
SArray* pTasks; SArray* pTasks;
int32_t taskDone; int32_t taskDone;
SMetaData jobRes; SMetaData jobRes;
int32_t rspCode;
uint64_t queryId; uint64_t queryId;
SCatalog* pCtg; SCatalog* pCtg;
...@@ -201,11 +200,12 @@ typedef struct SCtgMsgCtx { ...@@ -201,11 +200,12 @@ typedef struct SCtgMsgCtx {
typedef struct SCtgTask { typedef struct SCtgTask {
CTG_TASK_TYPE type; CTG_TASK_TYPE type;
int32_t taskId; int32_t taskId;
SCtgJob *pJob; SCtgJob* pJob;
void* taskCtx; void* taskCtx;
SCtgMsgCtx msgCtx; SCtgMsgCtx msgCtx;
void* res; int32_t code;
void* res;
} SCtgTask; } SCtgTask;
typedef int32_t (*ctgLanchTaskFp)(SCtgTask*); typedef int32_t (*ctgLanchTaskFp)(SCtgTask*);
......
...@@ -437,13 +437,14 @@ _return: ...@@ -437,13 +437,14 @@ _return:
int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableMeta) { if (NULL == pJob->jobRes.pTableMeta) {
pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, POINTER_BYTES); pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pTableMeta) { if (NULL == pJob->jobRes.pTableMeta) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
} }
taosArrayPush(pJob->jobRes.pTableMeta, &pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pTableMeta, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -451,14 +452,14 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { ...@@ -451,14 +452,14 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
int32_t ctgDumpDbVgRes(SCtgTask* pTask) { int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDbVgroup) { if (NULL == pJob->jobRes.pDbVgroup) {
pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, POINTER_BYTES); pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pDbVgroup) { if (NULL == pJob->jobRes.pDbVgroup) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
} }
taosArrayPush(pJob->jobRes.pDbVgroup, &pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
pTask->res = NULL; taosArrayPush(pJob->jobRes.pDbVgroup, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -466,13 +467,14 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) { ...@@ -466,13 +467,14 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
int32_t ctgDumpTbHashRes(SCtgTask* pTask) { int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableHash) { if (NULL == pJob->jobRes.pTableHash) {
pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SVgroupInfo)); pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pTableHash) { if (NULL == pJob->jobRes.pTableHash) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
} }
taosArrayPush(pJob->jobRes.pTableHash, pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pTableHash, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -480,21 +482,29 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { ...@@ -480,21 +482,29 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
int32_t ctgDumpIndexRes(SCtgTask* pTask) { int32_t ctgDumpIndexRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pIndex) { if (NULL == pJob->jobRes.pIndex) {
pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SIndexInfo)); pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pIndex) { if (NULL == pJob->jobRes.pIndex) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
} }
taosArrayPush(pJob->jobRes.pIndex, pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pIndex, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgDumpQnodeRes(SCtgTask* pTask) { int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pQnodeList) {
pJob->jobRes.pQnodeList = taosArrayInit(1, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pQnodeList) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
TSWAP(pJob->jobRes.pQnodeList, pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pQnodeList, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -502,13 +512,14 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) { ...@@ -502,13 +512,14 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDbCfg) { if (NULL == pJob->jobRes.pDbCfg) {
pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SDbCfgInfo)); pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pDbCfg) { if (NULL == pJob->jobRes.pDbCfg) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
} }
taosArrayPush(pJob->jobRes.pDbCfg, pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pDbCfg, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -516,13 +527,14 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { ...@@ -516,13 +527,14 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDbInfo) { if (NULL == pJob->jobRes.pDbInfo) {
pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SDbInfo)); pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pDbInfo) { if (NULL == pJob->jobRes.pDbInfo) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
} }
taosArrayPush(pJob->jobRes.pDbInfo, pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pDbInfo, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -530,13 +542,14 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { ...@@ -530,13 +542,14 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
int32_t ctgDumpUdfRes(SCtgTask* pTask) { int32_t ctgDumpUdfRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pUdfList) { if (NULL == pJob->jobRes.pUdfList) {
pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SFuncInfo)); pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pUdfList) { if (NULL == pJob->jobRes.pUdfList) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
} }
taosArrayPush(pJob->jobRes.pUdfList, pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pUdfList, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -544,13 +557,14 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) { ...@@ -544,13 +557,14 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) {
int32_t ctgDumpUserRes(SCtgTask* pTask) { int32_t ctgDumpUserRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pUser) { if (NULL == pJob->jobRes.pUser) {
pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(bool)); pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(SMetaRes));
if (NULL == pJob->jobRes.pUser) { if (NULL == pJob->jobRes.pUser) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
} }
taosArrayPush(pJob->jobRes.pUser, pTask->res); SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pUser, &res);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -559,20 +573,13 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { ...@@ -559,20 +573,13 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
int32_t code = 0; int32_t code = 0;
qDebug("QID:%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); qDebug("QID:0x%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
if (rspCode) { pTask->code = rspCode;
int32_t lastCode = atomic_val_compare_exchange_32(&pJob->rspCode, 0, rspCode);
if (0 == lastCode) {
CTG_ERR_JRET(rspCode);
}
return TSDB_CODE_SUCCESS;
}
int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1);
if (taskDone < taosArrayGetSize(pJob->pTasks)) { if (taskDone < taosArrayGetSize(pJob->pTasks)) {
qDebug("task done: %d, total: %d", taskDone, (int32_t)taosArrayGetSize(pJob->pTasks)); qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
...@@ -47,11 +47,13 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { ...@@ -47,11 +47,13 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) {
void ctgFreeSMetaData(SMetaData* pData) { void ctgFreeSMetaData(SMetaData* pData) {
taosArrayDestroy(pData->pTableMeta); taosArrayDestroy(pData->pTableMeta);
pData->pTableMeta = NULL; pData->pTableMeta = NULL;
/*
for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) {
SArray** pArray = taosArrayGet(pData->pDbVgroup, i); SArray** pArray = taosArrayGet(pData->pDbVgroup, i);
taosArrayDestroy(*pArray); taosArrayDestroy(*pArray);
} }
*/
taosArrayDestroy(pData->pDbVgroup); taosArrayDestroy(pData->pDbVgroup);
pData->pDbVgroup = NULL; pData->pDbVgroup = NULL;
...@@ -61,10 +63,12 @@ void ctgFreeSMetaData(SMetaData* pData) { ...@@ -61,10 +63,12 @@ void ctgFreeSMetaData(SMetaData* pData) {
taosArrayDestroy(pData->pUdfList); taosArrayDestroy(pData->pUdfList);
pData->pUdfList = NULL; pData->pUdfList = NULL;
/*
for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) {
SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i); SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i);
taosArrayDestroy(pInfo->pRetensions); taosArrayDestroy(pInfo->pRetensions);
} }
*/
taosArrayDestroy(pData->pDbCfg); taosArrayDestroy(pData->pDbCfg);
pData->pDbCfg = NULL; pData->pDbCfg = NULL;
...@@ -320,8 +324,12 @@ void ctgFreeTask(SCtgTask* pTask) { ...@@ -320,8 +324,12 @@ void ctgFreeTask(SCtgTask* pTask) {
break; break;
} }
case CTG_TASK_GET_DB_CFG: { case CTG_TASK_GET_DB_CFG: {
taosMemoryFreeClear(pTask->taskCtx); taosMemoryFreeClear(pTask->taskCtx);
taosMemoryFreeClear(pTask->res); if (pTask->res) {
SDbCfgInfo* pInfo = (SDbCfgInfo*)pTask->res;
taosArrayDestroy(pInfo->pRetensions);
taosMemoryFreeClear(pTask->res);
}
break; break;
} }
case CTG_TASK_GET_DB_INFO: { case CTG_TASK_GET_DB_INFO: {
......
...@@ -115,9 +115,9 @@ typedef struct SExplainCtx { ...@@ -115,9 +115,9 @@ typedef struct SExplainCtx {
#define EXPLAIN_ROW_NEW(level, ...) \ #define EXPLAIN_ROW_NEW(level, ...) \
do { \ do { \
if (isVerboseLine) { \ if (isVerboseLine) { \
tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 2 + 3, ""); \ tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 3 + 3, ""); \
} else { \ } else { \
tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 2, "", "-> "); \ tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 3, "", "-> "); \
} \ } \
tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__); \ tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__); \
} while (0) } while (0)
......
...@@ -28,91 +28,94 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { ...@@ -28,91 +28,94 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
} }
} }
// todo : to convert data according to SSDatablock static SSDataBlock* buildDescResultDataBlock() {
static void buildRspData(const STableMeta* pMeta, char* pData) { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
int32_t* payloadLen = (int32_t*) pData; pBlock->info.numOfCols = DESCRIBE_RESULT_COLS;
uint64_t* groupId = (uint64_t*)(pData + sizeof(int32_t)); pBlock->info.hasVarCol = true;
int32_t* pColSizes = (int32_t*)(pData + sizeof(int32_t) + sizeof(uint64_t)); pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
pData = (char*) pColSizes + DESCRIBE_RESULT_COLS * sizeof(int32_t);
int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta); SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN;
// Field taosArrayPush(pBlock->pDataBlock, &infoData);
int32_t* pOffset = (int32_t*)pData;
pData += numOfRows * sizeof(int32_t); infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
return pBlock;
}
static void setDescResultIntoDataBlock(SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) {
blockDataEnsureCapacity(pBlock, numOfRows);
pBlock->info.rows = numOfRows;
// field
SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
char buf[DESCRIBE_RESULT_FIELD_LEN] = {0};
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
STR_TO_VARSTR(pData, pMeta->schema[i].name); STR_TO_VARSTR(buf, pMeta->schema[i].name);
int16_t len = varDataTLen(pData); colDataAppend(pCol1, i, buf, false);
pData += len;
*pOffset = pColSizes[0];
pOffset += 1;
pColSizes[0] += len;
} }
// Type // Type
pOffset = (int32_t*)pData; SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
pData += numOfRows * sizeof(int32_t);
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
STR_TO_VARSTR(pData, tDataTypes[pMeta->schema[i].type].name); STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name);
int16_t len = varDataTLen(pData); colDataAppend(pCol2, i, buf, false);
pData += len;
*pOffset = pColSizes[1];
pOffset += 1;
pColSizes[1] += len;
} }
// Length // Length
pData += BitmapLen(numOfRows); SColumnInfoData* pCol3 = taosArrayGet(pBlock->pDataBlock, 2);
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
*(int32_t*)pData = getSchemaBytes(pMeta->schema + i); int32_t bytes = getSchemaBytes(pMeta->schema + i);
pData += sizeof(int32_t); colDataAppend(pCol3, i, (const char*)&bytes, false);
} }
pColSizes[2] = sizeof(int32_t) * numOfRows;
// Note // Note
pOffset = (int32_t*)pData; SColumnInfoData* pCol4 = taosArrayGet(pBlock->pDataBlock, 3);
pData += numOfRows * sizeof(int32_t);
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
STR_TO_VARSTR(pData, i >= pMeta->tableInfo.numOfColumns ? "TAG" : ""); STR_TO_VARSTR(buf, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
int16_t len = varDataTLen(pData); colDataAppend(pCol4, i, buf, false);
pData += len;
*pOffset = pColSizes[3];
pOffset += 1;
pColSizes[3] += len;
}
for (int32_t i = 0; i < DESCRIBE_RESULT_COLS; ++i) {
pColSizes[i] = htonl(pColSizes[i]);
} }
*payloadLen = (int32_t)(pData - (char*)payloadLen);
}
static int32_t calcRspSize(const STableMeta* pMeta) {
int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
return sizeof(SRetrieveTableRsp) +
(numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_FIELD_LEN) +
(numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_TYPE_LEN) +
(BitmapLen(numOfRows) + numOfRows * sizeof(int32_t)) +
(numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN) +
sizeof(int32_t) + sizeof(uint64_t);
} }
static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) {
SDescribeStmt* pDesc = (SDescribeStmt*)pStmt; SDescribeStmt* pDesc = (SDescribeStmt*) pStmt;
*pRsp = taosMemoryCalloc(1, calcRspSize(pDesc->pMeta)); int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta);
SSDataBlock* pBlock = buildDescResultDataBlock();
setDescResultIntoDataBlock(pBlock, numOfRows, pDesc->pMeta);
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) { if (NULL == *pRsp) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
(*pRsp)->useconds = 0; (*pRsp)->useconds = 0;
(*pRsp)->completed = 1; (*pRsp)->completed = 1;
(*pRsp)->precision = 0; (*pRsp)->precision = 0;
(*pRsp)->compressed = 0; (*pRsp)->compressed = 0;
(*pRsp)->compLen = 0; (*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htonl(TABLE_TOTAL_COL_NUM(pDesc->pMeta)); (*pRsp)->numOfRows = htonl(numOfRows);
buildRspData(pDesc->pMeta, (*pRsp)->data); (*pRsp)->numOfCols = htonl(DESCRIBE_RESULT_COLS);
int32_t len = 0;
blockCompressEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false);
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
blockDataDestroy(pBlock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tdatablock.h"
#include "commandInt.h" #include "commandInt.h"
#include "plannodes.h" #include "plannodes.h"
#include "query.h" #include "query.h"
...@@ -657,7 +658,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i ...@@ -657,7 +658,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
if (EXPLAIN_MODE_ANALYZE == ctx->mode) { if (EXPLAIN_MODE_ANALYZE == ctx->mode) {
// sort key // sort key
EXPLAIN_ROW_NEW(level, "Sort Key: "); EXPLAIN_ROW_NEW(level + 1, "Sort Key: ");
if (pResNode->pExecInfo) { if (pResNode->pExecInfo) {
for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) { for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) {
SOrderByExprNode *ptn = nodesListGetNode(pSortNode->pSortKeys, i); SOrderByExprNode *ptn = nodesListGetNode(pSortNode->pSortKeys, i);
...@@ -669,7 +670,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i ...@@ -669,7 +670,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
// sort method // sort method
EXPLAIN_ROW_NEW(level, "Sort Method: "); EXPLAIN_ROW_NEW(level + 1, "Sort Method: ");
int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo);
SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, 0); SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, 0);
...@@ -965,9 +966,32 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { ...@@ -965,9 +966,32 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
int32_t colNum = 1; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + SColumnInfoData infoData = {0};
sizeof(int32_t) * rowNum + pCtx->dataSize; infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
taosArrayPush(pBlock->pDataBlock, &infoData);
SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
pInfoData->hasNull = false;
colInfoDataEnsureCapacity(pInfoData, 0, rowNum);
char buf[1024] = {0};
for (int32_t i = 0; i < rowNum; ++i) {
SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
varDataCopy(buf, row->buf);
ASSERT(varDataTLen(row->buf) == row->len);
colDataAppend(pInfoData, i, buf, false);
}
pBlock->info.numOfCols = 1;
pBlock->info.rows = rowNum;
pBlock->info.hasVarCol = true;
int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
if (NULL == rsp) { if (NULL == rsp) {
qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
...@@ -977,34 +1001,13 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { ...@@ -977,34 +1001,13 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
rsp->completed = 1; rsp->completed = 1;
rsp->numOfRows = htonl(rowNum); rsp->numOfRows = htonl(rowNum);
// payload length int32_t len = 0;
*(int32_t *)rsp->data = blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0);
sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
// group id rsp->compLen = htonl(len);
*(uint64_t *)(rsp->data + sizeof(int32_t)) = 0;
// column length
int32_t *colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t));
// varchar column offset segment
int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t));
// varchar data real payload
char *data = (char *)(offset + rowNum);
char *start = data;
for (int32_t i = 0; i < rowNum; ++i) {
SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
offset[i] = data - start;
varDataCopy(data, row->buf);
ASSERT(varDataTLen(row->buf) == row->len);
data += row->len;
}
*colLength = htonl(data - start); blockDataDestroy(pBlock);
rsp->compLen = htonl(rspSize);
*pRsp = rsp; *pRsp = rsp;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
......
...@@ -353,6 +353,7 @@ typedef struct STagScanInfo { ...@@ -353,6 +353,7 @@ typedef struct STagScanInfo {
int32_t curPos; int32_t curPos;
SReadHandle readHandle; SReadHandle readHandle;
STableListInfo *pTableList; STableListInfo *pTableList;
SNode* pFilterNode; // filter info,
} STagScanInfo; } STagScanInfo;
typedef enum EStreamScanMode { typedef enum EStreamScanMode {
...@@ -687,6 +688,10 @@ typedef struct SSortedMergeOperatorInfo { ...@@ -687,6 +688,10 @@ typedef struct SSortedMergeOperatorInfo {
int32_t numOfResPerPage; int32_t numOfResPerPage;
char** groupVal; char** groupVal;
SArray *groupInfo; SArray *groupInfo;
bool hasGroupId;
uint64_t groupId;
STupleHandle* prefetchedTuple;
} SSortedMergeOperatorInfo; } SSortedMergeOperatorInfo;
typedef struct SSortOperatorInfo { typedef struct SSortOperatorInfo {
...@@ -699,6 +704,10 @@ typedef struct SSortOperatorInfo { ...@@ -699,6 +704,10 @@ typedef struct SSortOperatorInfo {
int64_t startTs; // sort start time int64_t startTs; // sort start time
uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included.
STupleHandle *prefetchedTuple;
bool hasGroupId;
uint64_t groupId;
} SSortOperatorInfo; } SSortOperatorInfo;
typedef struct STagFilterOperatorInfo { typedef struct STagFilterOperatorInfo {
...@@ -747,7 +756,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan ...@@ -747,7 +756,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
void doSetOperatorCompleted(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree);
SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset);
void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols);
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
...@@ -758,7 +767,7 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData ...@@ -758,7 +767,7 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData
SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
SColumn extractColumnFromColumnNode(SColumnNode* pColNode); SColumn extractColumnFromColumnNode(SColumnNode* pColNode);
SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo); SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SSortOperatorInfo* pInfo);
SSDataBlock* loadNextDataBlock(void* param); SSDataBlock* loadNextDataBlock(void* param);
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
...@@ -777,7 +786,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR ...@@ -777,7 +786,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
......
...@@ -68,10 +68,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { ...@@ -68,10 +68,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
} }
// data format: // data format:
// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ // +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... // |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
// | | (4 bytes) |(8 bytes) | sizeof(int32_t) * numOfCols | actual size | | actual size | | // | | (4 bytes) |(8 bytes) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | actual size | |
// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ // +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
// The length of bitmap is decided by number of rows of this data block, and the length of each column data is // The length of bitmap is decided by number of rows of this data block, and the length of each column data is
// recorded in the first segment, next to the struct header // recorded in the first segment, next to the struct header
static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
......
...@@ -1818,9 +1818,9 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO ...@@ -1818,9 +1818,9 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
} }
} }
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree) {
if (pFilterNode == NULL) { if (pFilterNode == NULL) {
return; return;
} }
...@@ -1839,11 +1839,11 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { ...@@ -1839,11 +1839,11 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols);
filterFreeInfo(filter); filterFreeInfo(filter);
extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); extractQualifiedTupleByFilterResult(pBlock, rowRes, keep, needFree);
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
} }
void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree) {
if (keep) { if (keep) {
return; return;
} }
...@@ -1883,8 +1883,20 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR ...@@ -1883,8 +1883,20 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
ASSERT(pBlock->info.rows == numOfRows); ASSERT(pBlock->info.rows == numOfRows);
} }
SColumnInfoData tmp = *pSrc;
*pSrc = *pDst; *pSrc = *pDst;
*pDst = tmp;
if (!needFree) {
if (IS_VAR_DATA_TYPE(pDst->info.type)) { // this elements do not need free
pDst->varmeta.offset = NULL;
} else {
pDst->nullbitmap = NULL;
}
pDst->pData = NULL;
}
} }
blockDataDestroy(px); // fix memory leak
} else { } else {
// do nothing // do nothing
pBlock->info.rows = 0; pBlock->info.rows = 0;
...@@ -2559,37 +2571,7 @@ int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadIn ...@@ -2559,37 +2571,7 @@ int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadIn
} }
} }
blockDataEnsureCapacity(pBlock, numOfRows); blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
int32_t dataLen = *(int32_t*)pStart;
uint64_t groupId = *(uint64_t*)(pStart + sizeof(int32_t));
pStart += sizeof(int32_t) + sizeof(uint64_t);
int32_t* colLen = (int32_t*)(pStart);
pStart += sizeof(int32_t) * numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) {
colLen[i] = htonl(colLen[i]);
ASSERT(colLen[i] >= 0);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
pColInfoData->varmeta.length = colLen[i];
pColInfoData->varmeta.allocLen = colLen[i];
memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows);
pStart += sizeof(int32_t) * numOfRows;
pColInfoData->pData = taosMemoryMalloc(colLen[i]);
} else {
memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
pStart += BitmapLen(numOfRows);
}
memcpy(pColInfoData->pData, pStart, colLen[i]);
pStart += colLen[i];
}
// data from mnode // data from mnode
relocateColumnData(pRes, pColList, pBlock->pDataBlock); relocateColumnData(pRes, pColList, pBlock->pDataBlock);
taosArrayDestroy(pBlock->pDataBlock); taosArrayDestroy(pBlock->pDataBlock);
...@@ -3121,6 +3103,68 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { ...@@ -3121,6 +3103,68 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) {
return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL; return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL;
} }
SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
SSortedMergeOperatorInfo *pInfo) {
blockDataCleanup(pDataBlock);
SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
if (p == NULL) {
return NULL;
}
blockDataEnsureCapacity(p, capacity);
while (1) {
STupleHandle* pTupleHandle = NULL;
if (pInfo->prefetchedTuple == NULL) {
pTupleHandle = tsortNextTuple(pHandle);
} else {
pTupleHandle = pInfo->prefetchedTuple;
pInfo->groupId = tsortGetGroupId(pTupleHandle);
pInfo->prefetchedTuple = NULL;
}
if (pTupleHandle == NULL) {
break;
}
uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
if (!pInfo->hasGroupId) {
pInfo->groupId = tupleGroupId;
pInfo->hasGroupId = true;
appendOneRowToDataBlock(p, pTupleHandle);
} else if (pInfo->groupId == tupleGroupId) {
appendOneRowToDataBlock(p, pTupleHandle);
} else {
pInfo->prefetchedTuple = pTupleHandle;
break;
}
if (p->info.rows >= capacity) {
break;
}
}
if (p->info.rows > 0) {
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows);
}
pDataBlock->info.rows = p->info.rows;
pDataBlock->info.capacity = p->info.rows;
pDataBlock->info.groupId = pInfo->groupId;
}
blockDataDestroy(p);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}
static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
...@@ -3129,7 +3173,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { ...@@ -3129,7 +3173,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSortedMergeOperatorInfo* pInfo = pOperator->info; SSortedMergeOperatorInfo* pInfo = pOperator->info;
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL); return getSortedMergeBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL, pInfo);
} }
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
...@@ -3640,7 +3684,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { ...@@ -3640,7 +3684,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
longjmp(pTaskInfo->env, code); longjmp(pTaskInfo->env, code);
} }
doFilter(pProjectInfo->pFilterNode, pBlock); doFilter(pProjectInfo->pFilterNode, pBlock, true);
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false); setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false);
blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
...@@ -4522,7 +4566,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo ...@@ -4522,7 +4566,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
STimeWindowAggSupp twSup = { STimeWindowAggSupp twSup = {
.waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
tsdbReaderT pDataReader = NULL; tsdbReaderT pDataReader = NULL;
if (pHandle->vnode) { if (pHandle->vnode) {
pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);
......
...@@ -359,7 +359,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { ...@@ -359,7 +359,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
while(1) { while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pRes); doFilter(pInfo->pCondition, pRes, true);
bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo); bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) { if (!hasRemain) {
......
...@@ -267,7 +267,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca ...@@ -267,7 +267,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
} }
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
doFilter(pTableScanInfo->pFilterNode, pBlock); doFilter(pTableScanInfo->pFilterNode, pBlock, false);
int64_t et = taosGetTimestampMs(); int64_t et = taosGetTimestampMs();
pTableScanInfo->readRecorder.filterTime += (et - st); pTableScanInfo->readRecorder.filterTime += (et - st);
...@@ -737,8 +737,8 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) { ...@@ -737,8 +737,8 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
SSDataBlock* pSDB = pInfo->pUpdateRes; SSDataBlock* pSDB = pInfo->pUpdateRes;
STimeWindow win = { STimeWindow win = {
.skey = INT64_MIN, .skey = INT64_MIN,
.ekey = INT64_MAX, .ekey = INT64_MAX,
}; };
bool needRead = false; bool needRead = false;
if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) { if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) {
...@@ -950,7 +950,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { ...@@ -950,7 +950,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes);
} }
doFilter(pInfo->pCondition, pInfo->pRes); doFilter(pInfo->pCondition, pInfo->pRes, false);
blockDataUpdateTsWindow(pInfo->pRes, 0); blockDataUpdateTsWindow(pInfo->pRes, 0);
break; break;
} }
...@@ -1722,7 +1722,9 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { ...@@ -1722,7 +1722,9 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
} }
pRes->info.rows = count; pRes->info.rows = count;
pOperator->resultInfo.totalRows += count; doFilter(pInfo->pFilterNode, pRes, true);
pOperator->resultInfo.totalRows += pRes->info.rows;
return (pRes->info.rows == 0) ? NULL : pInfo->pRes; return (pRes->info.rows == 0) ? NULL : pInfo->pRes;
} }
......
...@@ -42,6 +42,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR ...@@ -42,6 +42,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
pInfo->pSortInfo = pSortInfo; pInfo->pSortInfo = pSortInfo;
pInfo->pColMatchInfo = pColMatchColInfo; pInfo->pColMatchInfo = pColMatchColInfo;
pInfo->hasGroupId = false;
pInfo->prefetchedTuple = NULL;
pOperator->name = "SortOperator"; pOperator->name = "SortOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
pOperator->blocking = true; pOperator->blocking = true;
...@@ -81,8 +83,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { ...@@ -81,8 +83,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
pBlock->info.rows += 1; pBlock->info.rows += 1;
} }
SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
SArray* pColMatchInfo) { SSortOperatorInfo* pInfo) {
blockDataCleanup(pDataBlock); blockDataCleanup(pDataBlock);
SSDataBlock* p = tsortGetSortedDataBlock(pHandle); SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
...@@ -93,14 +95,33 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i ...@@ -93,14 +95,33 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
blockDataEnsureCapacity(p, capacity); blockDataEnsureCapacity(p, capacity);
while (1) { while (1) {
STupleHandle* pTupleHandle = tsortNextTuple(pHandle); STupleHandle* pTupleHandle = NULL;
if (pInfo->prefetchedTuple == NULL) {
pTupleHandle = tsortNextTuple(pHandle);
} else {
pTupleHandle = pInfo->prefetchedTuple;
pInfo->groupId = tsortGetGroupId(pTupleHandle);
pInfo->prefetchedTuple = NULL;
}
if (pTupleHandle == NULL) { if (pTupleHandle == NULL) {
break; break;
} }
appendOneRowToDataBlock(p, pTupleHandle); uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
if (!pInfo->hasGroupId) {
pInfo->groupId = tupleGroupId;
pInfo->hasGroupId = true;
appendOneRowToDataBlock(p, pTupleHandle);
} else if (pInfo->groupId == tupleGroupId) {
appendOneRowToDataBlock(p, pTupleHandle);
} else {
pInfo->prefetchedTuple = pTupleHandle;
break;
}
if (p->info.rows >= capacity) { if (p->info.rows >= capacity) {
return pDataBlock; break;
} }
} }
...@@ -117,6 +138,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i ...@@ -117,6 +138,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
pDataBlock->info.rows = p->info.rows; pDataBlock->info.rows = p->info.rows;
pDataBlock->info.capacity = p->info.rows; pDataBlock->info.capacity = p->info.rows;
pDataBlock->info.groupId = pInfo->groupId;
} }
blockDataDestroy(p); blockDataDestroy(p);
...@@ -188,8 +210,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { ...@@ -188,8 +210,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
longjmp(pTaskInfo->env, code); longjmp(pTaskInfo->env, code);
} }
SSDataBlock* pBlock = SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity,
getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); pInfo->pColMatchInfo, pInfo);
if (pBlock != NULL) { if (pBlock != NULL) {
pOperator->resultInfo.totalRows += pBlock->info.rows; pOperator->resultInfo.totalRows += pBlock->info.rows;
...@@ -230,11 +252,11 @@ typedef struct SMultiwaySortMergeOperatorInfo { ...@@ -230,11 +252,11 @@ typedef struct SMultiwaySortMergeOperatorInfo {
SArray* pColMatchInfo; // for index map from table scan output SArray* pColMatchInfo; // for index map from table scan output
SSDataBlock* pInputBlock; SSDataBlock* pInputBlock;
int64_t startTs; // sort start time int64_t startTs; // sort start time
bool hasGroupId; bool hasGroupId;
uint64_t groupId; uint64_t groupId;
STupleHandle *prefetchedTuple; STupleHandle* prefetchedTuple;
} SMultiwaySortMergeOperatorInfo; } SMultiwaySortMergeOperatorInfo;
int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
...@@ -274,7 +296,7 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { ...@@ -274,7 +296,7 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
} }
SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) { SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) {
blockDataCleanup(pDataBlock); blockDataCleanup(pDataBlock);
SSDataBlock* p = tsortGetSortedDataBlock(pHandle); SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
...@@ -285,12 +307,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData ...@@ -285,12 +307,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataEnsureCapacity(p, capacity); blockDataEnsureCapacity(p, capacity);
while (1) { while (1) {
STupleHandle* pTupleHandle = NULL; STupleHandle* pTupleHandle = NULL;
if (pInfo->prefetchedTuple == NULL) { if (pInfo->prefetchedTuple == NULL) {
pTupleHandle = tsortNextTuple(pHandle); pTupleHandle = tsortNextTuple(pHandle);
} else { } else {
pTupleHandle = pInfo->prefetchedTuple; pTupleHandle = pInfo->prefetchedTuple;
pInfo->groupId = tsortGetGroupId(pTupleHandle);
pInfo->prefetchedTuple = NULL; pInfo->prefetchedTuple = NULL;
} }
...@@ -307,14 +329,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData ...@@ -307,14 +329,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
appendOneRowToDataBlock(p, pTupleHandle); appendOneRowToDataBlock(p, pTupleHandle);
} else { } else {
pInfo->prefetchedTuple = pTupleHandle; pInfo->prefetchedTuple = pTupleHandle;
pInfo->groupId = tupleGroupId;
break; break;
} }
if (p->info.rows >= capacity) { if (p->info.rows >= capacity) {
break; break;
} }
} }
if (p->info.rows > 0) { if (p->info.rows > 0) {
...@@ -330,13 +350,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData ...@@ -330,13 +350,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
pDataBlock->info.rows = p->info.rows; pDataBlock->info.rows = p->info.rows;
pDataBlock->info.capacity = p->info.rows; pDataBlock->info.capacity = p->info.rows;
pDataBlock->info.groupId = pInfo->groupId;
} }
blockDataDestroy(p); blockDataDestroy(p);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
} }
SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
...@@ -350,12 +370,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { ...@@ -350,12 +370,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
longjmp(pTaskInfo->env, code); longjmp(pTaskInfo->env, code);
} }
SSDataBlock* pBlock = SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
getMultiwaySortedBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pInfo);
pInfo->binfo.pRes,
pOperator->resultInfo.capacity,
pInfo->pColMatchInfo,
pInfo);
if (pBlock != NULL) { if (pBlock != NULL) {
pOperator->resultInfo.totalRows += pBlock->info.rows; pOperator->resultInfo.totalRows += pBlock->info.rows;
...@@ -366,7 +382,7 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { ...@@ -366,7 +382,7 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
} }
void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
SMultiwaySortMergeOperatorInfo * pInfo = (SMultiwaySortMergeOperatorInfo*)param; SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock); pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock);
...@@ -386,9 +402,9 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx ...@@ -386,9 +402,9 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock, SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo,
SExecTaskInfo* pTaskInfo) { SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
int32_t rowSize = pResBlock->info.rowSize; int32_t rowSize = pResBlock->info.rowSize;
...@@ -412,7 +428,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, ...@@ -412,7 +428,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams,
pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2; pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2;
pInfo->sortBufSize = pInfo->bufPageSize * 16; pInfo->sortBufSize = pInfo->bufPageSize * 16;
pInfo->hasGroupId = false;
pInfo->prefetchedTuple = NULL;
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = pOperator->fpSet =
createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL,
......
...@@ -1227,7 +1227,10 @@ void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int ...@@ -1227,7 +1227,10 @@ void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int
SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
SResultRowPosition* p1 = SResultRowPosition* p1 =
(SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
ASSERT(p1); if (!p1) {
// window has been closed
return;
}
doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput); doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput);
} }
...@@ -2202,12 +2205,12 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { ...@@ -2202,12 +2205,12 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
} }
} }
int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo,
SDiskbasedBuf* pResultBuf) { int32_t numOfCols, SSDataBlock* pResultBlock) {
pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset); pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset);
pBasicInfo->pRes = pResultBlock; pBasicInfo->pRes = pResultBlock;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
pBasicInfo->pCtx[i].pBuf = pResultBuf; pBasicInfo->pCtx[i].pBuf = NULL;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -2237,16 +2240,15 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SEx ...@@ -2237,16 +2240,15 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SEx
initResultSizeInfo(pOperator, 4096); initResultSizeInfo(pOperator, 4096);
code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo"); code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf); code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo");
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
if (pInfo->pDummyCtx == NULL) { if (pInfo->pDummyCtx == NULL) {
...@@ -3101,6 +3103,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys ...@@ -3101,6 +3103,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
...@@ -3121,17 +3124,18 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys ...@@ -3121,17 +3124,18 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
.winMap = NULL, .winMap = NULL,
}; };
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
int32_t code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf); pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
if (pInfo->pDummyCtx == NULL) { if (pInfo->pDummyCtx == NULL) {
goto _error; goto _error;
......
...@@ -144,7 +144,9 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource ...@@ -144,7 +144,9 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource
(*sourceId) += 1; (*sourceId) += 1;
int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock);
int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock))/rowSize; // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
// The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize;
ASSERT(numOfRows > 0); ASSERT(numOfRows > 0);
return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows);
} }
...@@ -359,6 +361,11 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { ...@@ -359,6 +361,11 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
SSDataBlock* pLeftBlock = pLeftSource->src.pBlock; SSDataBlock* pLeftBlock = pLeftSource->src.pBlock;
SSDataBlock* pRightBlock = pRightSource->src.pBlock; SSDataBlock* pRightBlock = pRightSource->src.pBlock;
// first sort by block groupId
if (pLeftBlock->info.groupId != pRightBlock->info.groupId) {
return pLeftBlock->info.groupId < pRightBlock->info.groupId ? -1 : 1;
}
for(int32_t i = 0; i < pInfo->size; ++i) { for(int32_t i = 0; i < pInfo->size; ++i) {
SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId); SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
...@@ -521,16 +528,24 @@ static int32_t createInitialSources(SSortHandle* pHandle) { ...@@ -521,16 +528,24 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
if (pHandle->type == SORT_SINGLESOURCE_SORT) { if (pHandle->type == SORT_SINGLESOURCE_SORT) {
SSortSource* source = taosArrayGetP(pHandle->pOrderedSource, 0); SSortSource* source = taosArrayGetP(pHandle->pOrderedSource, 0);
taosArrayClear(pHandle->pOrderedSource); taosArrayClear(pHandle->pOrderedSource);
bool hasGroupId = false;
SSDataBlock* prefetchedDataBlock = NULL;
while (1) { while (1) {
SSDataBlock* pBlock = pHandle->fetchfp(source->param); SSDataBlock* pBlock = NULL;
if (prefetchedDataBlock == NULL) {
pBlock = pHandle->fetchfp(source->param);
} else {
pBlock = prefetchedDataBlock;
prefetchedDataBlock = NULL;
}
if (pBlock == NULL) { if (pBlock == NULL) {
break; break;
} }
if (pHandle->pDataBlock == NULL) { if (!hasGroupId) {
pHandle->pDataBlock = createOneDataBlock(pBlock, false);
// calculate the buffer pages according to the total available buffers. // calculate the buffer pages according to the total available buffers.
int32_t rowSize = blockDataGetRowSize(pBlock); int32_t rowSize = blockDataGetRowSize(pBlock);
if (rowSize * 4 > 4096) { if (rowSize * 4 > 4096) {
...@@ -542,29 +557,36 @@ static int32_t createInitialSources(SSortHandle* pHandle) { ...@@ -542,29 +557,36 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
// todo!! // todo!!
pHandle->numOfPages = 1024; pHandle->numOfPages = 1024;
sortBufSize = pHandle->numOfPages * pHandle->pageSize; sortBufSize = pHandle->numOfPages * pHandle->pageSize;
}
// perform the scalar function calculation before apply the sort hasGroupId = true;
if (pHandle->beforeFp != NULL) { pHandle->pDataBlock = createOneDataBlock(pBlock, false);
pHandle->beforeFp(pBlock, pHandle->param);
} }
// todo relocate the columns if (pHandle->pDataBlock->info.groupId == pBlock->info.groupId) {
int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock); // perform the scalar function calculation before apply the sort
if (code != 0) { if (pHandle->beforeFp != NULL) {
return code; pHandle->beforeFp(pBlock, pHandle->param);
} }
// todo relocate the columns
int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock);
if (code != 0) {
return code;
}
size_t size = blockDataGetSize(pHandle->pDataBlock); size_t size = blockDataGetSize(pHandle->pDataBlock);
if (size > sortBufSize) { if (size > sortBufSize) {
// Perform the in-memory sort and then flush data in the buffer into disk. // Perform the in-memory sort and then flush data in the buffer into disk.
int64_t p = taosGetTimestampUs(); int64_t p = taosGetTimestampUs();
blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
int64_t el = taosGetTimestampUs() - p; int64_t el = taosGetTimestampUs() - p;
pHandle->sortElapsed += el; pHandle->sortElapsed += el;
doAddToBuf(pHandle->pDataBlock, pHandle); doAddToBuf(pHandle->pDataBlock, pHandle);
}
} else {
prefetchedDataBlock = pBlock;
pHandle->pDataBlock = createOneDataBlock(pBlock, false);
} }
} }
......
...@@ -23,9 +23,13 @@ extern "C" { ...@@ -23,9 +23,13 @@ extern "C" {
#include "function.h" #include "function.h"
#include "functionMgt.h" #include "functionMgt.h"
bool dummyGetEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* UNUSED_PARAM(pEnv));
bool dummyInit(SqlFunctionCtx* UNUSED_PARAM(pCtx), SResultRowEntryInfo* UNUSED_PARAM(pResultInfo));
int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx));
int32_t dummyFinalize(SqlFunctionCtx* UNUSED_PARAM(pCtx), SSDataBlock* UNUSED_PARAM(pBlock));
bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx));
int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult); int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
...@@ -77,7 +81,11 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); ...@@ -77,7 +81,11 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool apercentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool apercentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t apercentileFunction(SqlFunctionCtx *pCtx); int32_t apercentileFunction(SqlFunctionCtx *pCtx);
int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx);
int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getApercentileMaxSize();
bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
...@@ -98,7 +106,10 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); ...@@ -98,7 +106,10 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t spreadFunction(SqlFunctionCtx* pCtx); int32_t spreadFunction(SqlFunctionCtx* pCtx);
int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx);
int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t getSpreadInfoSize();
bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
...@@ -108,7 +119,10 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); ...@@ -108,7 +119,10 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t histogramFunction(SqlFunctionCtx* pCtx); int32_t histogramFunction(SqlFunctionCtx* pCtx);
int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx);
int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t getHistogramInfoSize();
bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t hllFunction(SqlFunctionCtx* pCtx); int32_t hllFunction(SqlFunctionCtx* pCtx);
......
...@@ -251,6 +251,72 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t ...@@ -251,6 +251,72 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateApercentileImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (isPartial) {
if (2 != numOfParams && 3 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
// param1
SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1);
if (nodeType(pParamNode1) != QUERY_NODE_VALUE) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
SValueNode* pValue = (SValueNode*)pParamNode1;
if (pValue->datum.i < 0 || pValue->datum.i > 100) {
return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
}
pValue->notReserved = true;
uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
if (!IS_NUMERIC_TYPE(para1Type) || !IS_INTEGER_TYPE(para2Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
// param2
if (3 == numOfParams) {
uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type;
if (!IS_VAR_DATA_TYPE(para3Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
SNode* pParamNode2 = nodesListGetNode(pFunc->pParameterList, 2);
if (QUERY_NODE_VALUE != nodeType(pParamNode2) || !validateApercentileAlgo((SValueNode*)pParamNode2)) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
"Third parameter algorithm of apercentile must be 'default' or 't-digest'");
}
pValue = (SValueNode*)pParamNode2;
pValue->notReserved = true;
}
pFunc->node.resType = (SDataType){.bytes = getApercentileMaxSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
} else {
if (1 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (TSDB_DATA_TYPE_BINARY != para1Type) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateApercentilePartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateApercentileImpl(pFunc, pErrBuf, len, true);
}
static int32_t translateApercentileMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateApercentileImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters // pseudo column do not need to check parameters
pFunc->node.resType = pFunc->node.resType =
...@@ -311,6 +377,34 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) ...@@ -311,6 +377,34 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateSpreadImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (isPartial) {
if (!IS_NUMERIC_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
} else {
if (TSDB_DATA_TYPE_BINARY != paraType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateSpreadPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateSpreadImpl(pFunc, pErrBuf, len, true);
}
static int32_t translateSpreadMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateSpreadImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (1 != numOfParams && 2 != numOfParams) { if (1 != numOfParams && 2 != numOfParams) {
...@@ -409,6 +503,58 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l ...@@ -409,6 +503,58 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (isPartial) {
if (4 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_NUMERIC_TYPE(colType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
// param1 ~ param3
for (int32_t i = 1; i < numOfParams; ++i) {
SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
if (QUERY_NODE_VALUE != nodeType(pParamNode)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
SValueNode* pValue = (SValueNode*)pParamNode;
pValue->notReserved = true;
}
if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
} else {
if (1 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type != TSDB_DATA_TYPE_BINARY) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = 512, .type = TSDB_DATA_TYPE_BINARY};
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateHistogramPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateHistogramImpl(pFunc, pErrBuf, len, true);
}
static int32_t translateHistogramMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateHistogramImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) { if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
...@@ -1056,9 +1202,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1056,9 +1202,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.processFunc = countFunction, .processFunc = countFunction,
.finalizeFunc = functionFinalize, .finalizeFunc = functionFinalize,
.invertFunc = countInvertFunction, .invertFunc = countInvertFunction,
.combineFunc = combineFunction, .combineFunc = combineFunction,
// .pPartialFunc = "count", .pPartialFunc = "count",
// .pMergeFunc = "sum" .pMergeFunc = "sum"
}, },
{ {
.name = "sum", .name = "sum",
...@@ -1071,7 +1217,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1071,7 +1217,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.processFunc = sumFunction, .processFunc = sumFunction,
.finalizeFunc = functionFinalize, .finalizeFunc = functionFinalize,
.invertFunc = sumInvertFunction, .invertFunc = sumInvertFunction,
.combineFunc = sumCombine, .combineFunc = sumCombine,
.pPartialFunc = "sum",
.pMergeFunc = "sum"
}, },
{ {
.name = "min", .name = "min",
...@@ -1083,7 +1231,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1083,7 +1231,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = minmaxFunctionSetup, .initFunc = minmaxFunctionSetup,
.processFunc = minFunction, .processFunc = minFunction,
.finalizeFunc = minmaxFunctionFinalize, .finalizeFunc = minmaxFunctionFinalize,
.combineFunc = minCombine .combineFunc = minCombine,
.pPartialFunc = "min",
.pMergeFunc = "min"
}, },
{ {
.name = "max", .name = "max",
...@@ -1095,7 +1245,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1095,7 +1245,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = minmaxFunctionSetup, .initFunc = minmaxFunctionSetup,
.processFunc = maxFunction, .processFunc = maxFunction,
.finalizeFunc = minmaxFunctionFinalize, .finalizeFunc = minmaxFunctionFinalize,
.combineFunc = maxCombine .combineFunc = maxCombine,
.pPartialFunc = "max",
.pMergeFunc = "max"
}, },
{ {
.name = "stddev", .name = "stddev",
...@@ -1141,7 +1293,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1141,7 +1293,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.getEnvFunc = getPercentileFuncEnv, .getEnvFunc = getPercentileFuncEnv,
.initFunc = percentileFunctionSetup, .initFunc = percentileFunctionSetup,
.processFunc = percentileFunction, .processFunc = percentileFunction,
.finalizeFunc = percentileFinalize .finalizeFunc = percentileFinalize,
.invertFunc = NULL,
.combineFunc = NULL,
}, },
{ {
.name = "apercentile", .name = "apercentile",
...@@ -1151,6 +1305,29 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1151,6 +1305,29 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.getEnvFunc = getApercentileFuncEnv, .getEnvFunc = getApercentileFuncEnv,
.initFunc = apercentileFunctionSetup, .initFunc = apercentileFunctionSetup,
.processFunc = apercentileFunction, .processFunc = apercentileFunction,
.finalizeFunc = apercentileFinalize,
.combineFunc = apercentileCombine,
.pPartialFunc = "_apercentile_partial",
.pMergeFunc = "_apercentile_merge"
},
{
.name = "_apercentile_partial",
.type = FUNCTION_TYPE_APERCENTILE_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateApercentilePartial,
.getEnvFunc = getApercentileFuncEnv,
.initFunc = apercentileFunctionSetup,
.processFunc = apercentileFunction,
.finalizeFunc = apercentilePartialFinalize
},
{
.name = "_apercentile_merge",
.type = FUNCTION_TYPE_APERCENTILE_MERGE,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateApercentileMerge,
.getEnvFunc = getApercentileFuncEnv,
.initFunc = functionSetup,
.processFunc = apercentileFunctionMerge,
.finalizeFunc = apercentileFinalize .finalizeFunc = apercentileFinalize
}, },
{ {
...@@ -1182,6 +1359,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1182,6 +1359,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.getEnvFunc = getSpreadFuncEnv, .getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup, .initFunc = spreadFunctionSetup,
.processFunc = spreadFunction, .processFunc = spreadFunction,
.finalizeFunc = spreadFinalize,
.pPartialFunc = "_spread_partial",
.pMergeFunc = "_spread_merge"
},
{
.name = "_spread_partial",
.type = FUNCTION_TYPE_SPREAD_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateSpreadPartial,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup,
.processFunc = spreadFunction,
.finalizeFunc = spreadPartialFinalize
},
{
.name = "_spread_merge",
.type = FUNCTION_TYPE_SPREAD_MERGE,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateSpreadMerge,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup,
.processFunc = spreadFunctionMerge,
.finalizeFunc = spreadFinalize .finalizeFunc = spreadFinalize
}, },
{ {
...@@ -1214,7 +1415,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1214,7 +1415,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = functionSetup, .initFunc = functionSetup,
.processFunc = firstFunction, .processFunc = firstFunction,
.finalizeFunc = firstLastFinalize, .finalizeFunc = firstLastFinalize,
.combineFunc = firstCombine, .combineFunc = firstCombine,
}, },
{ {
.name = "last", .name = "last",
...@@ -1245,6 +1446,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -1245,6 +1446,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.getEnvFunc = getHistogramFuncEnv, .getEnvFunc = getHistogramFuncEnv,
.initFunc = histogramFunctionSetup, .initFunc = histogramFunctionSetup,
.processFunc = histogramFunction, .processFunc = histogramFunction,
.finalizeFunc = histogramFinalize,
.pPartialFunc = "_histogram_partial",
.pMergeFunc = "_histogram_merge"
},
{
.name = "_histogram_partial",
.type = FUNCTION_TYPE_HISTOGRAM_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateHistogramPartial,
.getEnvFunc = getHistogramFuncEnv,
.initFunc = histogramFunctionSetup,
.processFunc = histogramFunction,
.finalizeFunc = histogramPartialFinalize
},
{
.name = "_histogram_merge",
.type = FUNCTION_TYPE_HISTOGRAM_MERGE,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateHistogramMerge,
.getEnvFunc = getHistogramFuncEnv,
.initFunc = functionSetup,
.processFunc = histogramFunctionMerge,
.finalizeFunc = histogramFinalize .finalizeFunc = histogramFinalize
}, },
{ {
......
...@@ -70,7 +70,7 @@ void indexCacheBroadcast(void* cache); ...@@ -70,7 +70,7 @@ void indexCacheBroadcast(void* cache);
void indexCacheWait(void* cache); void indexCacheWait(void* cache);
Iterate* indexCacheIteratorCreate(IndexCache* cache); Iterate* indexCacheIteratorCreate(IndexCache* cache);
void indexCacheIteratorDestroy(Iterate* iiter); void idxCacheIteratorDestroy(Iterate* iiter);
int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid); int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid);
...@@ -82,7 +82,7 @@ void indexCacheUnRef(IndexCache* cache); ...@@ -82,7 +82,7 @@ void indexCacheUnRef(IndexCache* cache);
void indexCacheDebug(IndexCache* cache); void indexCacheDebug(IndexCache* cache);
void indexCacheDestroyImm(IndexCache* cache); void idxCacheDestroyImm(IndexCache* cache);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
...@@ -26,8 +26,9 @@ extern "C" { ...@@ -26,8 +26,9 @@ extern "C" {
extern char JSON_COLUMN[]; extern char JSON_COLUMN[];
extern char JSON_VALUE_DELIM; extern char JSON_VALUE_DELIM;
char* indexPackJsonData(SIndexTerm* itm); char* idxPackJsonData(SIndexTerm* itm);
char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip);
typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
...@@ -39,13 +40,14 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b); ...@@ -39,13 +40,14 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b);
_cache_range_compare indexGetCompare(RangeType ty); _cache_range_compare indexGetCompare(RangeType ty);
int32_t indexConvertData(void* src, int8_t type, void** dst); int32_t idxConvertData(void* src, int8_t type, void** dst);
int32_t indexConvertDataToStr(void* src, int8_t type, void** dst); int32_t idxConvertDataToStr(void* src, int8_t type, void** dst);
int32_t indexGetDataByteLen(int8_t type); int32_t idxGetDataByteLen(int8_t type);
char* indexInt2str(int64_t val, char* dst, int radix); char* idxInt2str(int64_t val, char* dst, int radix);
int idxUidCompare(const void* a, const void* b);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
...@@ -29,9 +29,9 @@ extern "C" { ...@@ -29,9 +29,9 @@ extern "C" {
#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a)
typedef struct Fst Fst; typedef struct Fst Fst;
typedef struct FstNode FstNode; typedef struct FstNode FstNode;
typedef struct StreamWithState StreamWithState; typedef struct FStmSt FStmSt;
typedef enum { Included, Excluded, Unbounded } FstBound; typedef enum { Included, Excluded, Unbounded } FstBound;
...@@ -40,12 +40,12 @@ typedef struct FstBoundWithData { ...@@ -40,12 +40,12 @@ typedef struct FstBoundWithData {
FstBound type; FstBound type;
} FstBoundWithData; } FstBoundWithData;
typedef struct FstStreamBuilder { typedef struct FStmBuilder {
Fst* fst; Fst* fst;
AutomationCtx* aut; FAutoCtx* aut;
FstBoundWithData* min; FstBoundWithData* min;
FstBoundWithData* max; FstBoundWithData* max;
} FstStreamBuilder, FstStreamWithStateBuilder; } FStmBuilder, FStmStBuilder;
typedef struct FstRange { typedef struct FstRange {
uint64_t start; uint64_t start;
...@@ -267,17 +267,17 @@ typedef struct Fst { ...@@ -267,17 +267,17 @@ typedef struct Fst {
Fst* fstCreate(FstSlice* data); Fst* fstCreate(FstSlice* data);
void fstDestroy(Fst* fst); void fstDestroy(Fst* fst);
bool fstGet(Fst* fst, FstSlice* b, Output* out); bool fstGet(Fst* fst, FstSlice* b, Output* out);
FstNode* fstGetNode(Fst* fst, CompiledAddr); FstNode* fstGetNode(Fst* fst, CompiledAddr);
FstNode* fstGetRoot(Fst* fst); FstNode* fstGetRoot(Fst* fst);
FstType fstGetType(Fst* fst); FstType fstGetType(Fst* fst);
CompiledAddr fstGetRootAddr(Fst* fst); CompiledAddr fstGetRootAddr(Fst* fst);
Output fstEmptyFinalOutput(Fst* fst, bool* null); Output fstEmptyFinalOutput(Fst* fst, bool* null);
FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx); FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx);
FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx); FStmStBuilder* fstSearchWithState(Fst* fst, FAutoCtx* ctx);
// into stream to expand later // into stream to expand later
StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb); FStmSt* stmBuilderIntoStm(FStmBuilder* sb);
bool fstVerify(Fst* fst); bool fstVerify(Fst* fst);
...@@ -293,41 +293,40 @@ typedef struct StreamState { ...@@ -293,41 +293,40 @@ typedef struct StreamState {
void streamStateDestroy(void* s); void streamStateDestroy(void* s);
typedef struct StreamWithState { typedef struct FStmSt {
Fst* fst; Fst* fst;
AutomationCtx* aut; FAutoCtx* aut;
SArray* inp; SArray* inp;
FstOutput emptyOutput; FstOutput emptyOutput;
SArray* stack; // <StreamState> SArray* stack; // <StreamState>
FstBoundWithData* endAt; FstBoundWithData* endAt;
} StreamWithState; } FStmSt;
typedef struct StreamWithStateResult { typedef struct FStmStRslt {
FstSlice data; FstSlice data;
FstOutput out; FstOutput out;
void* state; void* state;
} StreamWithStateResult; } FStmStRslt;
StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state); FStmStRslt* swsResultCreate(FstSlice* data, FstOutput fOut, void* state);
void swsResultDestroy(StreamWithStateResult* result); void swsResultDestroy(FStmStRslt* result);
typedef void* (*StreamCallback)(void*); typedef void* (*StreamCallback)(void*);
StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FStmSt* stmStCreate(Fst* fst, FAutoCtx* automation, FstBoundWithData* min, FstBoundWithData* max);
FstBoundWithData* max);
void streamWithStateDestroy(StreamWithState* sws); void stmStDestroy(FStmSt* sws);
bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min); bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min);
StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback); FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback);
FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut); FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut);
void fstStreamBuilderDestroy(FstStreamBuilder* b); void stmBuilderDestroy(FStmBuilder* b);
// set up bound range // set up bound range
// refator later: to simple code by marco // refator later: to simple code by marco
void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type); void stmBuilderSetRange(FStmBuilder* b, FstSlice* val, RangeType type);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -22,24 +22,24 @@ extern "C" { ...@@ -22,24 +22,24 @@ extern "C" {
#include "indexFstUtil.h" #include "indexFstUtil.h"
#include "indexInt.h" #include "indexInt.h"
typedef struct AutomationCtx AutomationCtx; typedef struct FAutoCtx FAutoCtx;
typedef enum AutomationType { AUTOMATION_ALWAYS, AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType; typedef enum AutomationType { AUTOMATION_ALWAYS, AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType;
typedef struct StartWith { typedef struct StartWith {
AutomationCtx* autoSelf; FAutoCtx* autoSelf;
} StartWith; } StartWith;
typedef struct Complement { typedef struct Complement {
AutomationCtx* autoSelf; FAutoCtx* autoSelf;
} Complement; } Complement;
// automation // automation
typedef struct AutomationCtx { typedef struct FAutoCtx {
AutomationType type; AutomationType type;
void* stdata; void* stdata;
char* data; char* data;
} AutomationCtx; } FAutoCtx;
typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType; typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType;
typedef enum StartWithStateKind { Done, Running } StartWithStateKind; typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
...@@ -60,16 +60,16 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv); ...@@ -60,16 +60,16 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv);
void startWithStateValueDestroy(void* sv); void startWithStateValueDestroy(void* sv);
typedef struct AutomationFunc { typedef struct AutomationFunc {
void* (*start)(AutomationCtx* ctx); void* (*start)(FAutoCtx* ctx);
bool (*isMatch)(AutomationCtx* ctx, void*); bool (*isMatch)(FAutoCtx* ctx, void*);
bool (*canMatch)(AutomationCtx* ctx, void* data); bool (*canMatch)(FAutoCtx* ctx, void* data);
bool (*willAlwaysMatch)(AutomationCtx* ctx, void* state); bool (*willAlwaysMatch)(FAutoCtx* ctx, void* state);
void* (*accept)(AutomationCtx* ctx, void* state, uint8_t byte); void* (*accept)(FAutoCtx* ctx, void* state, uint8_t byte);
void* (*acceptEof)(AutomationCtx* ct, void* state); void* (*acceptEof)(FAutoCtx* ct, void* state);
} AutomationFunc; } AutomationFunc;
AutomationCtx* automCtxCreate(void* data, AutomationType atype); FAutoCtx* automCtxCreate(void* data, AutomationType atype);
void automCtxDestroy(AutomationCtx* ctx); void automCtxDestroy(FAutoCtx* ctx);
extern AutomationFunc automFuncs[]; extern AutomationFunc automFuncs[];
#ifdef __cplusplus #ifdef __cplusplus
......
...@@ -43,7 +43,7 @@ extern "C" { ...@@ -43,7 +43,7 @@ extern "C" {
#define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0) #define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0)
// clang-format on // clang-format on
typedef enum { LT, LE, GT, GE, CONTAINS } RangeType; typedef enum { LT, LE, GT, GE, CONTAINS, EQ } RangeType;
typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef enum { kTypeValue, kTypeDeletion } STermValueType;
typedef struct SIndexStat { typedef struct SIndexStat {
......
此差异已折叠。
此差异已折叠。
此差异已折叠。
...@@ -173,10 +173,8 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { ...@@ -173,10 +173,8 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
param->colId = l->colId; param->colId = l->colId;
param->colValType = l->node.resType.type; param->colValType = l->node.resType.type;
memcpy(param->dbName, l->dbName, sizeof(l->dbName)); memcpy(param->dbName, l->dbName, sizeof(l->dbName));
#pragma GCC diagnostic push memcpy(param->colName, r->literal, strlen(r->literal));
#pragma GCC diagnostic ignored "-Wformat-overflow" // sprintf(param->colName, "%s_%s", l->colName, r->literal);
sprintf(param->colName, "%s_%s", l->colName, r->literal);
#pragma GCC diagnostic pop
param->colValType = r->typeData; param->colValType = r->typeData;
return 0; return 0;
// memcpy(param->colName, l->colName, sizeof(l->colName)); // memcpy(param->colName, l->colName, sizeof(l->colName));
...@@ -188,6 +186,9 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { ...@@ -188,6 +186,9 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue)); SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
param->colId = -1; param->colId = -1;
param->colValType = (uint8_t)(vn->node.resType.type); param->colValType = (uint8_t)(vn->node.resType.type);
if (vn->literal == NULL || strlen(vn->literal) == 0) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
memcpy(param->colName, vn->literal, strlen(vn->literal)); memcpy(param->colName, vn->literal, strlen(vn->literal));
break; break;
} }
...@@ -340,9 +341,9 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { ...@@ -340,9 +341,9 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
return NULL; return NULL;
} }
static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
SIndexMetaArg *arg = &output->arg; int ret = 0;
int ret = 0;
SIndexMetaArg * arg = &output->arg;
EIndexQueryType qtype = 0; EIndexQueryType qtype = 0;
SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
if (left->colValType == TSDB_DATA_TYPE_JSON) { if (left->colValType == TSDB_DATA_TYPE_JSON) {
...@@ -506,7 +507,6 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { ...@@ -506,7 +507,6 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
if (nParam <= 1) { if (nParam <= 1) {
output->status = SFLT_NOT_INDEX; output->status = SFLT_NOT_INDEX;
return code; return code;
// SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
if (node->opType == OP_TYPE_JSON_GET_VALUE) { if (node->opType == OP_TYPE_JSON_GET_VALUE) {
return code; return code;
...@@ -547,6 +547,8 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou ...@@ -547,6 +547,8 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou
} else if (node->condType == LOGIC_COND_TYPE_NOT) { } else if (node->condType == LOGIC_COND_TYPE_NOT) {
// taosArrayAddAll(output->result, params[m].result); // taosArrayAddAll(output->result, params[m].result);
} }
taosArraySort(output->result, idxUidCompare);
taosArrayRemoveDuplicate(output->result, idxUidCompare, NULL);
} }
} else { } else {
for (int32_t m = 0; m < node->pParameterList->length; m++) { for (int32_t m = 0; m < node->pParameterList->length; m++) {
......
此差异已折叠。
...@@ -22,6 +22,14 @@ int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { ...@@ -22,6 +22,14 @@ int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) {
int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
for (int i = 0; i < taosArrayGetSize(terms); i++) { for (int i = 0; i < taosArrayGetSize(terms); i++) {
SIndexJsonTerm *p = taosArrayGetP(terms, i); SIndexJsonTerm *p = taosArrayGetP(terms, i);
if (p->colType == TSDB_DATA_TYPE_BOOL) {
p->colType = TSDB_DATA_TYPE_INT;
} else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR ||
p->colType == TSDB_DATA_TYPE_BINARY) {
// p->colType = TSDB_DATA_TYPE_NCHAR;
} else {
p->colType = TSDB_DATA_TYPE_DOUBLE;
}
INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
} }
// handle put // handle put
...@@ -32,6 +40,14 @@ int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *re ...@@ -32,6 +40,14 @@ int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *re
SArray *terms = tq->query; SArray *terms = tq->query;
for (int i = 0; i < taosArrayGetSize(terms); i++) { for (int i = 0; i < taosArrayGetSize(terms); i++) {
SIndexJsonTerm *p = taosArrayGetP(terms, i); SIndexJsonTerm *p = taosArrayGetP(terms, i);
if (p->colType == TSDB_DATA_TYPE_BOOL) {
p->colType = TSDB_DATA_TYPE_INT;
} else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR ||
p->colType == TSDB_DATA_TYPE_BINARY) {
// p->colType = TSDB_DATA_TYPE_NCHAR;
} else {
p->colType = TSDB_DATA_TYPE_DOUBLE;
}
INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
} }
// handle search // handle search
......
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
...@@ -327,13 +327,13 @@ TEST_F(UtilEnv, testFill) { ...@@ -327,13 +327,13 @@ TEST_F(UtilEnv, testFill) {
for (int i = 0; i < 1000000; i++) { for (int i = 0; i < 1000000; i++) {
int64_t val = i; int64_t val = i;
char buf[65] = {0}; char buf[65] = {0};
indexInt2str(val, buf, 1); idxInt2str(val, buf, 1);
EXPECT_EQ(val, taosStr2int64(buf)); EXPECT_EQ(val, taosStr2int64(buf));
} }
for (int i = 0; i < 1000000; i++) { for (int i = 0; i < 1000000; i++) {
int64_t val = 0 - i; int64_t val = 0 - i;
char buf[65] = {0}; char buf[65] = {0};
indexInt2str(val, buf, -1); idxInt2str(val, buf, -1);
EXPECT_EQ(val, taosStr2int64(buf)); EXPECT_EQ(val, taosStr2int64(buf));
} }
} }
......
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册