diff --git a/.gitignore b/.gitignore index ba8543e518e1bfe6a8b479cd6acae21b50b9b226..2308ea78969849379d019f18b238ce496dbdc29d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ debug/ release/ target/ debs/ +deps/ rpms/ mac/ *.pyc @@ -100,4 +101,4 @@ TAGS contrib/* !contrib/CMakeLists.txt -!contrib/test +!contrib/test \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b626977588441c53da8564d2396eb0da158df4ae..1a8cb1d710028be170a1249b7cf4b2baccaa7ec1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") # contrib add_subdirectory(contrib) @@ -20,9 +20,13 @@ add_library(api INTERFACE) target_include_directories(api INTERFACE "include/client") # src +if(${BUILD_TEST}) + include(CTest) + enable_testing() +endif(${BUILD_TEST}) add_subdirectory(source) # docs add_subdirectory(docs) -# tests (TODO) +# tests (TODO) \ No newline at end of file diff --git a/include/client/taos.h b/include/client/taos.h index 0f7edc9fedba746c1f5510063b0acc4bd8dea95b..19d191b84e7db8ed42430b9f82440b2b02075f06 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -73,10 +73,12 @@ typedef struct taosField { #define DLL_EXPORT #endif -DLL_EXPORT int taos_init(); +typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); + DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); @@ -154,14 +156,14 @@ DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); // TODO: the return value should be `const` -DLL_EXPORT char *taos_get_server_info(TAOS *taos); -DLL_EXPORT char *taos_get_client_info(); -DLL_EXPORT char *taos_errstr(TAOS_RES *tres); +DLL_EXPORT const char *taos_get_server_info(TAOS *taos); +DLL_EXPORT const char *taos_get_client_info(); +DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); DLL_EXPORT int taos_errno(TAOS_RES *tres); -DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param); -DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); +DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); +DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 906dd38c51796f7395f7720addca0efa9a3599e2..a75c4a7bac9555af40f667442eb45c12974c5e88 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -74,9 +74,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_DB, "compact-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_FUNCTION, "create-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE_FUNCTION, "retrieve-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE, "create-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE, "drop-stable" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) @@ -94,9 +94,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) // message from vnode to dnode // message from mnode to vnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE_IN, "create-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE_IN, "alter-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE_IN, "drop-stable" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-in" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-in" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-in" ) // message from mnode to mnode // message from mnode to qnode // message from mnode to dnode @@ -159,7 +159,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_DNODE, TSDB_MGMT_TABLE_MNODE, TSDB_MGMT_TABLE_VGROUP, - TSDB_MGMT_TABLE_STABLE, + TSDB_MGMT_TABLE_STB, TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, @@ -299,7 +299,7 @@ typedef struct { uint64_t superTableUid; uint64_t createdTime; char tableFname[TSDB_TABLE_FNAME_LEN]; - char stableFname[TSDB_TABLE_FNAME_LEN]; + char stbFname[TSDB_TABLE_FNAME_LEN]; char data[]; } SMDCreateTableMsg; @@ -316,16 +316,23 @@ typedef struct { } SCreateTableMsg; typedef struct { - int32_t numOfTables; - int32_t contLen; -} SCMCreateTableMsg; + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igExists; + int32_t numOfTags; + int32_t numOfColumns; + SSchema pSchema[]; +} SCreateStbMsg; typedef struct { char name[TSDB_TABLE_FNAME_LEN]; - // if user specify DROP STABLE, this flag will be set. And an error will be returned if it is not a super table - int8_t supertable; int8_t igNotExists; -} SCMDropTableMsg; +} SDropStbMsg; + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t alterType; + SSchema schema; +} SAlterStbMsg; typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; @@ -356,6 +363,7 @@ typedef struct { int32_t pid; char app[TSDB_APP_NAME_LEN]; char db[TSDB_DB_NAME_LEN]; + int64_t startTime; } SConnectMsg; typedef struct SEpSet { @@ -366,19 +374,17 @@ typedef struct SEpSet { } SEpSet; typedef struct { - int32_t acctId; - int32_t clusterId; - int32_t connId; - int8_t superAuth; - int8_t readAuth; - int8_t writeAuth; - int8_t reserved[5]; - SEpSet epSet; + int32_t acctId; + uint32_t clusterId; + int32_t connId; + int8_t superUser; + int8_t reserved[5]; + SEpSet epSet; } SConnectRsp; typedef struct { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; int32_t maxUsers; int32_t maxDbs; int32_t maxTimeSeries; @@ -393,7 +399,7 @@ typedef struct { typedef struct { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; } SCreateUserMsg, SAlterUserMsg; typedef struct { @@ -668,7 +674,6 @@ typedef struct { typedef struct { int32_t statusInterval; - int32_t mnodeEqualVnodeNum; int64_t checkTime; // 1970-01-01 00:00:00.000 char timezone[TSDB_TIMEZONE_LEN]; // tsTimezone char locale[TSDB_LOCALE_LEN]; // tsLocale @@ -694,7 +699,7 @@ typedef struct SStatusMsg { int32_t sver; int32_t dnodeId; int32_t clusterId; - uint32_t rebootTime; // time stamp for last reboot + int64_t rebootTime; // time stamp for last reboot int16_t numOfCores; int16_t numOfSupportMnodes; int16_t numOfSupportVnodes; @@ -770,7 +775,7 @@ typedef struct { typedef struct { char name[TSDB_TABLE_FNAME_LEN]; -} SStableInfoMsg; +} SStbInfoMsg; typedef struct { SMsgHead msgHead; @@ -815,8 +820,8 @@ typedef struct { } SVgroupsMsg, SVgroupsInfo; typedef struct { - char tableFname[TSDB_TABLE_FNAME_LEN]; // table id - char stableFname[TSDB_TABLE_FNAME_LEN]; + char tbFname[TSDB_TABLE_FNAME_LEN]; // table id + char stbFname[TSDB_TABLE_FNAME_LEN]; int32_t numOfTags; int32_t numOfColumns; int8_t precision; @@ -880,7 +885,7 @@ typedef struct { typedef struct { int32_t dnodeId; - char config[128]; + char config[TSDB_DNODE_CONFIG_LEN]; } SCfgDnodeMsg; typedef struct { @@ -973,8 +978,8 @@ typedef struct { char user[TSDB_USER_LEN]; char spi; char encrypt; - char secret[TSDB_KEY_LEN]; - char ckey[TSDB_KEY_LEN]; + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; } SAuthMsg, SAuthRsp; typedef struct { diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index a3de2452acff4148ff4a81223fcdd7e64797893d..c6ef6c513fd7cdec335ea8c0c9d8b51edaee85f6 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -544,7 +544,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value) { +static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx* pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); diff --git a/include/common/tep.h b/include/common/tep.h new file mode 100644 index 0000000000000000000000000000000000000000..2addc11e5a6251be8cfcb1ca4ef688edcfbde56f --- /dev/null +++ b/include/common/tep.h @@ -0,0 +1,17 @@ +#ifndef TDENGINE_TEP_H +#define TDENGINE_TEP_H + +#include "os.h" +#include "taosmsg.h" + +typedef struct SCorEpSet { + int32_t version; + SEpSet epSet; +} SCorEpSet; + +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); + +void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); + +#endif // TDENGINE_TEP_H diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e62083e9993718c72fb165f026e7334157ed8f39..a9b0e607614811281a63f62349b49b91ed15cb5d 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -81,8 +81,6 @@ extern int64_t tsMaxRetentWindow; // db parameters in client extern int32_t tsCacheBlockSize; extern int32_t tsBlocksPerVnode; -extern int32_t tsMinTablePerVnode; -extern int32_t tsMaxTablePerVnode; extern int32_t tsTableIncStepPerVnode; extern int32_t tsMaxVgroupsPerDb; extern int16_t tsDaysPerFile; @@ -108,22 +106,13 @@ extern int8_t tsEnableBalance; extern int8_t tsAlternativeRole; extern int32_t tsBalanceInterval; extern int32_t tsOfflineThreshold; -extern int32_t tsMnodeEqualVnodeNum; extern int8_t tsEnableFlowCtrl; extern int8_t tsEnableSlaveQuery; extern int8_t tsEnableAdjustMaster; // restful -extern int8_t tsEnableHttpModule; extern int32_t tsRestRowLimit; -extern uint16_t tsHttpPort; -extern int32_t tsHttpCacheSessions; -extern int32_t tsHttpSessionExpire; -extern int32_t tsHttpMaxThreads; -extern int8_t tsHttpEnableCompress; -extern int8_t tsHttpEnableRecordSql; extern int8_t tsTelegrafUseFieldNum; -extern int8_t tsHttpDbNameMandatory; // mqtt extern int8_t tsEnableMqttModule; @@ -145,7 +134,6 @@ extern int8_t tsEnableStream; // internal extern int8_t tsPrintAuth; -extern int8_t tscEmbedded; extern char tsVnodeDir[]; extern char tsMnodeDir[]; extern int64_t tsTickPerDay[3]; @@ -194,7 +182,7 @@ extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) void taosInitGlobalCfg(); -int32_t taosCheckGlobalCfg(); +int32_t taosCheckAndPrintCfg(); int32_t taosCfgDynamicOptions(char *msg); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); diff --git a/include/common/ttime.h b/include/common/ttime.h index 926d23c23994f7c726c2ac53d4aa230fa84baaeb..ed6496aca670bb04f001e9a6f610cd1951e0dec5 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -44,10 +44,10 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); -int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); +int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision); -int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); +int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); void deltaToUtcInitOnce(); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 552c7eaa3ffc7a01c2f3415653b12f39111879d5..e068c6eb6ca4cc56b3eae27d5ca213517ad86752 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -40,7 +40,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* bool taosVariantIsValid(SVariant *pVar); -void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type); +void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type); void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 540a501f3a8f8b03d7ca55cfeade21e30a8387c2..fbe447baf9d35bcd34c2ab0202d2e9dc849785b3 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -33,7 +33,6 @@ typedef struct { int16_t numOfSupportQnodes; int8_t enableTelem; int32_t statusInterval; - int32_t mnodeEqualVnodeNum; float numOfThreadsPerCore; float ratioOfQueryCores; int32_t maxShellConns; diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 8a8346fbc0eca5ba2fc978a71666a4497b2dac49..f4a6f7a4c93f778ce738ed0cf9ceae63bcb4b6e6 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -47,7 +47,6 @@ typedef struct SMnodeCfg { int32_t sver; int8_t enableTelem; int32_t statusInterval; - int32_t mnodeEqualVnodeNum; int32_t shellActivityTimer; char *timezone; char *locale; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index d1f7d9a958d93266b47d4b81d5093b14819169de..7e7afc97740f512e4afc8da2e620548bf367a7c9 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -158,8 +158,8 @@ typedef enum { SDB_USER = 5, SDB_AUTH = 6, SDB_ACCT = 7, - SDB_VGROUP = 9, - SDB_STABLE = 9, + SDB_VGROUP = 8, + SDB_STB = 9, SDB_DB = 10, SDB_FUNC = 11, SDB_MAX = 12 diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 85533f65bc25154ead1c8771f7f06c31fad0e3ce..7359df92cc680c7c052131e843ed158dcf4ff8f4 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -256,7 +256,7 @@ typedef struct STQ { // the collection of group handle // the handle of kvstore char* path; - STqCfg* tqConfig; + STqCfg* tqConfig; TqLogReader* tqLogReader; TqMemRef tqMemRef; TqMetaStore* tqMeta; diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 30531ad738bbabf82b3a672f6415da9793620497..007ce838128d6894aa89e8a07b783459806221ff 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -36,6 +36,8 @@ typedef struct SVnodeCfg { struct { /** write buffer size */ uint64_t wsize; + uint64_t ssize; + uint64_t lsize; /** use heap allocator or arena allocator */ bool isHeapAllocator; }; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2ceb893ce4b1293f4fa08e997e13b66641f24600..5019191fd195fb34ec8ff03bbbf5469dd81aa24e 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -44,12 +44,12 @@ typedef struct SCatalogReq { bool qNodeRequired; // valid qnode } SCatalogReq; -typedef struct SCatalogRsp { +typedef struct SMetaData { SArray *pTableMeta; // STableMeta array SArray *pVgroupInfo; // SVgroupInfo list SArray *pUdfList; // udf info list SEpSet *pEpSet; // qnode epset list -} SCatalogRsp; +} SMetaData; typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema @@ -135,7 +135,7 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe * @param pMetaData * @return */ -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SCatalogRsp* pRsp); +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 0e6c352d71e736dc08fca4e1d5e344180a99fda8..2f152c3e2b8f45cf8c9ccb80bc1fe45efd771da0 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -131,6 +131,18 @@ struct SInsertStmtInfo; */ bool qIsInsertSql(const char* pStr, size_t length); +typedef struct SParseContext { + const char* pSql; // sql string + size_t sqlLen; // length of the sql string + int64_t id; // operator id, generated by uuid generator + const char* pDbname; + const SEpSet* pEpSet; + int8_t schemaAttached; // denote if submit block is built with table schema or not + + char* pMsg; // extended error message if exists to help avoid the problem in sql statement. + int32_t msgLen; // max length of the msg +} SParseContext; + /** * Parse the sql statement and then return the SQueryStmtInfo as the result of bounded AST. * @param pSql sql string @@ -141,16 +153,35 @@ bool qIsInsertSql(const char* pStr, size_t length); */ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen); +typedef enum { + PAYLOAD_TYPE_KV = 0, + PAYLOAD_TYPE_RAW = 1, +} EPayloadType; + +typedef struct SVgDataBlocks { + int64_t vgId; // virtual group id + int32_t numOfTables; // number of tables in current submit block + uint32_t size; + char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... +} SVgDataBlocks; + +typedef struct SInsertStmtInfo { + SArray* pDataBlocks; // data block for each vgroup, SArray. + int8_t schemaAttache; // denote if submit block is built with table schema or not + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* sql; // current sql statement position +} SInsertStmtInfo; + /** * Parse the insert sql statement. * @param pStr sql string * @param length length of the sql string - * @param pInsertParam data in binary format to submit to vnode directly. * @param id operator id, generated by uuid generator. * @param msg extended error message if exists to help avoid the problem in sql statement. - * @return + * @return data in binary format to submit to vnode directly. */ -int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen); + int32_t qParseInsertSql(SParseContext* pContext, struct SInsertStmtInfo** pInfo); /** * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 2a8a7aad4b05abc1c80edde9eee706e38607450e..2afd9d48b76839682764dab87fb1dd7adc7796be 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -29,11 +29,6 @@ extern "C" { extern int tsRpcHeadSize; -typedef struct SRpcCorEpSet { - int32_t version; - SEpSet epSet; -} SRpcCorEpSet; - typedef struct SRpcConnInfo { uint32_t clientIp; uint16_t clientPort; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index f84297670b376cebc3136b40cede02ca046d969e..744275e6ff66937ef0a61068bdb7e5aadf61f7da 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -32,12 +32,49 @@ extern int32_t wDebugFlag; #define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} #define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define WAL_PREFIX "wal" +#define WAL_PREFIX_LEN 3 +#define WAL_NOSUFFIX_LEN 20 +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) +#define WAL_LOG_SUFFIX "log" +#define WAL_INDEX_SUFFIX "idx" +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) + +#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) +#define WAL_CUR_POS_WRITABLE 1 +#define WAL_CUR_FILE_WRITABLE 2 +#define WAL_CUR_FAILED 4 + +#pragma pack(push,1) typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType; +typedef struct SWalReadHead { + int8_t sver; + uint8_t msgType; + int8_t reserved[2]; + int32_t len; + //int64_t ingestTs; //not implemented + int64_t version; + char body[]; +} SWalReadHead; + +typedef struct { + int32_t vgId; + int32_t fsyncPeriod; // millisecond + int32_t retentionPeriod; // secs + int32_t rollPeriod; // secs + int64_t retentionSize; + int64_t segSize; + EWalType level; // wal level +} SWalCfg; + typedef struct { //union { //uint32_t info; @@ -47,87 +84,53 @@ typedef struct { //uint32_t reserved : 24; //}; //}; - int8_t sver; - uint8_t msgType; - int8_t reserved[2]; - int32_t len; - int64_t version; - uint32_t signature; uint32_t cksumHead; uint32_t cksumBody; - char cont[]; + SWalReadHead head; } SWalHead; -typedef struct { - int32_t vgId; - int32_t fsyncPeriod; // millisecond - int32_t rollPeriod; - int64_t segSize; - EWalType walLevel; // wal level -} SWalCfg; - -#define WAL_PREFIX "wal" -#define WAL_PREFIX_LEN 3 -#define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) -#define WAL_LOG_SUFFIX "log" -#define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) -#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFEUL)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -//#define WAL_FILE_NUM 1 // 3 -#define WAL_FILESET_MAX 128 - -#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) -#define WAL_CUR_POS_WRITABLE 1 -#define WAL_CUR_FILE_WRITABLE 2 -#define WAL_CUR_FAILED 4 +typedef struct SWalVer { + int64_t firstVer; + int64_t verInSnapshotting; + int64_t snapshotVer; + int64_t commitVer; + int64_t lastVer; +} SWalVer; typedef struct SWal { // cfg - int32_t vgId; - int32_t fsyncPeriod; // millisecond - int32_t rollPeriod; // second - int64_t segSize; - int64_t retentionSize; - int32_t retentionPeriod; - EWalType level; - //total size - int64_t totSize; - //fsync seq - int32_t fsyncSeq; - //reference - int64_t refId; - //write tfd - int64_t writeLogTfd; - int64_t writeIdxTfd; - //read tfd - int64_t readLogTfd; - int64_t readIdxTfd; - //current version - int64_t curVersion; - //wal lifecycle - int64_t firstVersion; - int64_t snapshotVersion; - int64_t commitVersion; - int64_t lastVersion; - //roll status - int64_t lastRollSeq; + SWalCfg cfg; + SWalVer vers; //file set int32_t writeCur; - int32_t readCur; + int64_t writeLogTfd; + int64_t writeIdxTfd; SArray* fileInfoSet; //ctl int32_t curStatus; + int32_t fsyncSeq; + int64_t totSize; + int64_t refId; + int64_t lastRollSeq; pthread_mutex_t mutex; //path char path[WAL_PATH_LEN]; //reusable write head - SWalHead head; + SWalHead writeHead; } SWal; // WAL HANDLE +typedef struct SWalReadHandle { + SWal* pWal; + int64_t readLogTfd; + int64_t readIdxTfd; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t capacity; + int64_t status; //if cursor valid + SWalHead* pHead; +} SWalReadHandle; +#pragma pack(pop) + typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); // module initialization @@ -148,10 +151,15 @@ int32_t walCommit(SWal *, int64_t ver); // truncate after int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely -int32_t walTakeSnapshot(SWal *, int64_t ver); +int32_t walBeginTakeSnapshot(SWal *, int64_t ver); +int32_t walEndTakeSnapshot(SWal *); //int32_t walDataCorrupted(SWal*); // read +SWalReadHandle* walOpenReadHandle(SWal *); +void walCloseReadHandle(SWalReadHandle *); +int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); + int32_t walRead(SWal *, SWalHead **, int64_t ver); int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 86a9f10819f11d54cd860bfbc01ebb6ba9d34129..78112fc7a0f58d0659cdbd0043c47e814ca69ca0 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -57,7 +57,7 @@ int64_t taosGetPthreadId(pthread_t thread); void taosResetPthread(pthread_t* thread); bool taosComparePthread(pthread_t first, pthread_t second); int32_t taosGetPId(); -int32_t taosGetCurrentAPPName(char* name, int32_t* len); +int32_t taosGetAppName(char* name, int32_t* len); #ifdef __cplusplus } diff --git a/include/os/osString.h b/include/os/osString.h index 358324075fab292657a207c37c3d1292a137b0d9..582411d4440e7b1539c4b926413c7ed9751dfe62 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -38,11 +38,11 @@ extern "C" { (dst)[(size)-1] = 0; \ } while (0) -int64_t taosStr2int64(char *str); +int64_t taosStr2int64(const char *str); // USE_LIBICONV int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); -bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); +bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); bool taosValidateEncodec(const char *encodec); char * taosCharsetReplace(char *charsetstr); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e1a1694fab0dedf46fbc92f36b1ef775879a4a6a..8400031f9bdfcf0979725f85261d93426d4d502c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -153,22 +153,23 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x032A) #define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x032B) -#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) //"DNode already exists") -#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) //"DNode does not exist") +// mnode-dnode +#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) +#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) +#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0332) +#define TSDB_CODE_MND_INVALID_CLUSTER_CFG TAOS_DEF_ERROR_CODE(0, 0x0333) +#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x0334) +#define TSDB_CODE_MND_INVALID_DNODE_CFG TAOS_DEF_ERROR_CODE(0, 0x0335) +#define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0336) +#define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0337) + +// mnode-vgroup #define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0332) //"VGroup does not exist") -#define TSDB_CODE_MND_NO_REMOVE_MASTER TAOS_DEF_ERROR_CODE(0, 0x0333) //"Master DNode cannot be removed") -#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0334) //"Out of DNodes") -#define TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT TAOS_DEF_ERROR_CODE(0, 0x0335) //"Cluster cfg inconsistent") -#define TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION TAOS_DEF_ERROR_CODE(0, 0x0336) //"Invalid dnode cfg option") -#define TSDB_CODE_MND_BALANCE_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0337) //"Balance already enabled") #define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0338) //"Vgroup not in dnode") #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0339) //"Vgroup already in dnode") -#define TSDB_CODE_MND_DNODE_NOT_FREE TAOS_DEF_ERROR_CODE(0, 0x033A) //"Dnode not avaliable") -#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x033B) //"Cluster id not match") #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x033C) //"Cluster not ready") -#define TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED TAOS_DEF_ERROR_CODE(0, 0x033D) //"Dnode Id not configured") -#define TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED TAOS_DEF_ERROR_CODE(0, 0x033E) //"Dnode Ep not configured") +// mnode-acct #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) //"Account already exists") #define TSDB_CODE_MND_ACCT_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0341) //"Invalid account") #define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0342) //"Invalid account options") @@ -184,8 +185,18 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0348) //"Mnode already exists") #define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0349) //"Mnode not there") -// mnode-table -#define TSDB_CODE_MND_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) //"Table already exists") +// mnode-stable +#define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_IGEXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COLS_NUM TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COL_TYPE TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COL_ID TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COL_BYTES TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COL_NAME TAOS_DEF_ERROR_CODE(0, 0x0360) + + #define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) //"Table name too long") #define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") #define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) //"Invalid table type in tsdb") diff --git a/include/util/tarray.h b/include/util/tarray.h index 25862a7119f4a22e379fe4dfd6dcaf8ee6ac09f6..e0f14dcd25cb7ddec01cb9a12c23bc142b1852f6 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -43,6 +43,13 @@ typedef struct SArray { */ void* taosArrayInit(size_t size, size_t elemSize); +/** + * + * @param tsize + * @return + */ +int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize); + /** * * @param pArray @@ -153,6 +160,13 @@ void taosArraySet(SArray* pArray, size_t index, void* pData); */ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt); +/** + * remove some data entry from front + * @param pArray + * @param cnt + */ +void taosArrayPopTailBatch(SArray* pArray, size_t cnt); + /** * remove data entry of the given index * @param pArray @@ -213,6 +227,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn); */ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); +/** + * search the array, return index of the element + * @param pArray + * @param compar + * @param key + */ +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); + /** * search the array * @param pArray diff --git a/include/util/tchecksum.h b/include/util/tchecksum.h index c1907b0c464c657ee2aeebba7871d1dd786e7d40..fb6c85edcda5e70de8ba9ad18e8dfd1b9317e4ed 100644 --- a/include/util/tchecksum.h +++ b/include/util/tchecksum.h @@ -39,7 +39,7 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin } static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { - return (checksum == (*crc32c)(0, stream, (size_t)ssize)); + return (checksum != (*crc32c)(0, stream, (size_t)ssize)); } static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) { diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 15100423d5e4edac40a61f7436822cbeb1853aab..7dee01247b9e081585ab1ba4c4a1f907dec9cbc6 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 123 +#define TSDB_CFG_MAX_NUM 115 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 @@ -83,11 +83,11 @@ extern int32_t tsGlobalConfigNum; extern char * tsCfgStatusStr[]; void taosReadGlobalLogCfg(); -int32_t taosReadGlobalCfg(); -void taosPrintGlobalCfg(); +int32_t taosReadCfgFromFile(); +void taosPrintCfg(); void taosDumpGlobalCfg(); -void taosInitConfigOption(SGlobalCfg cfg); +void taosAddConfigOption(SGlobalCfg cfg); SGlobalCfg *taosGetConfigOption(const char *option); #ifdef __cplusplus diff --git a/include/util/tdef.h b/include/util/tdef.h index 3ab515671ab37d26f715a3fb7b037fdfe8bbbd30..300b190e97a9093298ae849664f688975fcec7fe 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -161,7 +161,7 @@ do { \ #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 33 +#define TSDB_DB_NAME_LEN 65 #define TSDB_FULL_DB_NAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN) #define TSDB_FUNC_NAME_LEN 65 @@ -193,7 +193,7 @@ do { \ #define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_AUTH_LEN 16 -#define TSDB_KEY_LEN 16 +#define TSDB_PASSWORD_LEN 32 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 @@ -209,6 +209,8 @@ do { \ #define TSDB_STEP_NAME_LEN 32 #define TSDB_STEP_DESC_LEN 128 +#define TSDB_DNODE_CONFIG_LEN 128 + #define TSDB_MQTT_HOSTNAME_LEN 64 #define TSDB_MQTT_PORT_LEN 8 #define TSDB_MQTT_USER_LEN 24 diff --git a/include/util/tdlist.h b/include/util/tdlist.h new file mode 100644 index 0000000000000000000000000000000000000000..a19f3bebec4e396f13c11349561a79be80f58a29 --- /dev/null +++ b/include/util/tdlist.h @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_TDLIST_H_ +#define _TD_UTIL_TDLIST_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Single linked list +#define TD_SLIST_NODE(TYPE) \ + struct { \ + struct TYPE *sl_next_; \ + } + +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + int sl_neles_; \ + } + +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) + +#define tSListInit(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + (sl)->sl_neles_ = 0; \ + } while (0) + +#define tSListPush(sl, sln) \ + do { \ + TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + +#define tSListPop(sl) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + +// Double linked list +#define TD_DLIST_NODE(TYPE) \ + struct { \ + TYPE *dl_prev_; \ + TYPE *dl_next_; \ + } + +#define TD_DLIST(TYPE) \ + struct { \ + struct TYPE *dl_head_; \ + struct TYPE *dl_tail_; \ + int dl_neles_; \ + } + +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) + +#define tDListInit(dl) \ + do { \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ + TD_DLIST_NELES(dl) = 0; \ + } while (0) + +#define tDListAppend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define tDListPrepend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = NULL; \ + TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define tDListPop(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ + } \ + if (TD_DLIST_NODE_PREV(dln) != NULL) { \ + TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ + TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + } while (0) + +#if 0 +// List iterator +#define TD_LIST_FITER 0 +#define TD_LIST_BITER 1 +#define TD_LIST_ITER(S) \ + struct { \ + int it_dir_; \ + S * it_next_; \ + S * it_ptr_; \ + TD_DLIST(S) * it_list_; \ + } + +#define tlistIterInit(it, l, dir) \ + (it)->it_dir_ = (dir); \ + (it)->it_list_ = l; \ + if ((dir) == TD_LIST_FITER) { \ + (it)->it_next_ = (l)->dl_head_; \ + } else { \ + (it)->it_next_ = (l)->dl_tail_; \ + } + +#define tlistIterNext(it) \ + ({ \ + (it)->it_ptr_ = (it)->it_next_; \ + if ((it)->it_next_ != NULL) { \ + if ((it)->it_dir_ == TD_LIST_FITER) { \ + (it)->it_next_ = (it)->it_next_->next_; \ + } else { \ + (it)->it_next_ = (it)->it_next_->prev_; \ + } \ + } \ + (it)->it_ptr_; \ + }) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_TDLIST_H_*/ \ No newline at end of file diff --git a/include/util/tmacro.h b/include/util/tmacro.h new file mode 100644 index 0000000000000000000000000000000000000000..74056cfe071366c8f93d2de93cad7a28169c6d01 --- /dev/null +++ b/include/util/tmacro.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_MACRO_H_ +#define _TD_UTIL_MACRO_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Module init/clear MACRO definitions +#define TD_MOD_UNINITIALIZED 0 +#define TD_MOD_INITIALIZED 1 + +#define TD_MOD_UNCLEARD 0 +#define TD_MOD_CLEARD 1 + +#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED +#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD + +#define TD_CHECK_AND_SET_MODE_INIT(MOD) \ + atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) + +#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file diff --git a/include/util/tutil.h b/include/util/tutil.h index e4e66901d34ec2fc56d41751dfc13fb5b3c1428c..c315948702af31b5dbcf8cde71331f132554532f 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -29,7 +29,7 @@ int32_t strdequote(char *src); int32_t strndequote(char *dst, const char *z, int32_t len); int32_t strRmquote(char *z, int32_t len); size_t strtrim(char *src); -char *strnchr(char *haystack, char needle, int32_t len, bool skipquote); +char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); char **strsplit(char *src, const char *delim, int32_t *num); char *strtolower(char *dst, const char *src); char *strntolower(char *dst, const char *src, int32_t n); @@ -45,14 +45,25 @@ char *taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); void taosIp2String(uint32_t ip, char *str); void taosIpPort2String(uint32_t ip, uint16_t port, char *str); -int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; tMD5Init(&context); tMD5Update(&context, inBuf, (unsigned int)inLen); tMD5Final(&context); - memcpy(target, context.digest, TSDB_KEY_LEN); + memcpy(target, context.digest, tListLen(context.digest)); +} + +static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) { + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, inBuf, (unsigned int)len); + tMD5Final(&context); + + sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], + context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], + context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], + context.digest[13], context.digest[14], context.digest[15]); } #ifdef __cplusplus diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index bc18e416646a335cb2d4ad80fd8ac8e6e066900c..b46afa31b9299cd3a41e68913181e03389d2a63c 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -2,10 +2,14 @@ aux_source_directory(src CLIENT_SRC) add_library(taos ${CLIENT_SRC}) target_include_directories( taos - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_SOURCE_DIR}/include/client" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( taos PRIVATE common INTERFACE api + PRIVATE os util common transport parser ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h new file mode 100644 index 0000000000000000000000000000000000000000..3180923aff03a8d99d470f38e4d000c9274fe878 --- /dev/null +++ b/source/client/inc/clientInt.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLIENTINT_H +#define TDENGINE_CLIENTINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taos.h" +#include "taosmsg.h" +#include "thash.h" +#include "tlist.h" +#include "trpc.h" +#include "tdef.h" +#include "tmsgtype.h" +#include "tep.h" + +typedef struct SQueryExecMetric { + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server +} SQueryExecMetric; + +typedef struct SInstanceActivity { + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. + + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj +} SInstanceActivity; + +typedef struct SHeartBeatInfo { + void *pTimer; // timer, used to send request msg to mnode +} SHeartBeatInfo; + +typedef struct SAppInstInfo { + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceActivity summary; + SList *pConnList; // STscObj linked list + uint32_t clusterId; + void *pTransporter; +} SAppInstInfo; + +typedef struct SAppInfo { + int64_t startTime; + char appName[TSDB_APP_NAME_LEN]; + char *ep; + int32_t pid; + int32_t numOfThreads; + SHeartBeatInfo hb; + SHashObj *pInstMap; +} SAppInfo; + +typedef struct STscObj { + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char acctId[TSDB_ACCT_ID_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + uint32_t connId; + uint64_t id; // ref ID returned by taosAddRef +// struct SSqlObj *sqlList; + void *pTransporter; + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj from this tscObj + SAppInstInfo *pAppInfo; +} STscObj; + +typedef struct SReqBody { + tsem_t rspSem; // not used now + void* fp; + void* param; +} SRequestBody; + +typedef struct SRequestObj { + uint64_t requestId; + int32_t type; // request type + STscObj *pTscObj; + SQueryExecMetric metric; + char *sqlstr; // sql string + SRequestBody body; + int64_t self; + char *msgBuf; + int32_t code; + void *pInfo; // sql parse info, generated by parser module +} SRequestObj; + +typedef struct SRequestMsgBody { + int32_t msgType; + void *pData; + int32_t msgLen; + uint64_t requestId; + uint64_t requestObjRefId; +} SRequestMsgBody; + +extern SAppInfo appInfo; +extern int32_t tscReqRef; +extern void *tscQhandle; +extern int32_t tscConnRef; + +extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg); +extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); + +int taos_init(); + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo); +void destroyTscObj(void*pObj); + +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void destroyRequest(SRequestObj* pRequest); + +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); + +void taos_init_imp(void); +int taos_options_imp(TSDB_OPTION option, const char *str); + +void* openTransporter(const char *user, const char *auth); + +void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); +void initMsgHandleFp(); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTINT_H diff --git a/source/client/inc/tscLog.h b/source/client/inc/tscLog.h new file mode 100644 index 0000000000000000000000000000000000000000..f205a50227308e71e27e9715e9f819078be5a20a --- /dev/null +++ b/source/client/inc/tscLog.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSCLOG_H +#define TDENGINE_TSCLOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/client/src/client.c b/source/client/src/client.c index 258fd9156503f49b7ed471b6abba8952c9ed8552..863e96aa744ee42f0b15fb0f4376bd4351b17220 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -13,14 +13,51 @@ * along with this program. If not, see . */ -//#include "taos.h" +#include "os.h" +#include "tdef.h" +#include "tglobal.h" +#include "clientInt.h" +#include "tscLog.h" -//TAOS_RES *taos_query(TAOS *taos, const char *sql) { -// -//} -#include "taosmsg.h" +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { + int32_t p = (port != 0)? port:tsServerPort; -int taos_init() { return 0; } -void taos_cleanup(void) {} + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + if (pass == NULL) { + pass = TSDB_DEFAULT_PASS; + } + + return taos_connect_internal(ip, user, pass, NULL, db, p); +} + +TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { + tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (auth == NULL) { + tscError("No auth info is given, failed to connect to server"); + return NULL; + } + + return taos_connect_internal(ip, user, NULL, auth, db, port); +} + +TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { + char ipStr[TSDB_EP_LEN] = {0}; + char dbStr[TSDB_DB_NAME_LEN] = {0}; + char userStr[TSDB_USER_LEN] = {0}; + char passStr[TSDB_PASSWORD_LEN] = {0}; + + strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + return taos_connect(ipStr, userStr, passStr, dbStr, port); +} diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c new file mode 100644 index 0000000000000000000000000000000000000000..2ae79f1947e9385ca5bb31cddc01be803e4d6d0d --- /dev/null +++ b/source/client/src/clientImpl.c @@ -0,0 +1,283 @@ +#include +#include "clientInt.h" +#include "tdef.h" +#include "tep.h" +#include "tglobal.h" +#include "tmsgtype.h" +#include "tref.h" +#include "tscLog.h" + +static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); +static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody); +static void destroyConnectMsg(SRequestMsgBody* pMsgBody); + +static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId); + +static bool stringLengthCheck(const char* str, size_t maxsize) { + if (str == NULL) { + return false; + } + + size_t len = strlen(str); + if (len <= 0 || len > maxsize) { + return false; + } + + return true; +} + +static bool validateUserName(const char* user) { + return stringLengthCheck(user, TSDB_USER_LEN - 1); +} + +static bool validatePassword(const char* passwd) { + return stringLengthCheck(passwd, TSDB_PASSWORD_LEN - 1); +} + +static bool validateDbName(const char* db) { + return stringLengthCheck(db, TSDB_DB_NAME_LEN - 1); +} + +static char* getClusterKey(const char* user, const char* auth, const char* ip, int32_t port) { + char key[512] = {0}; + snprintf(key, sizeof(key), "%s:%s:%s:%d", user, auth, ip, port); + return strdup(key); +} + +static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); + +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { + if (taos_init() != TSDB_CODE_SUCCESS) { + return NULL; + } + + if (!validateUserName(user)) { + terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; + return NULL; + } + + char tmp[TSDB_DB_NAME_LEN] = {0}; + if (db != NULL) { + if(!validateDbName(db)) { + terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; + return NULL; + } + + tstrncpy(tmp, db, sizeof(tmp)); + strdequote(tmp); + } + + char secretEncrypt[32] = {0}; + if (auth == NULL) { + if (!validatePassword(pass)) { + terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; + return NULL; + } + + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); + } else { + tstrncpy(secretEncrypt, auth, tListLen(secretEncrypt)); + } + + SCorEpSet epSet = {0}; + if (ip) { + if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { + return NULL; + } + + if (port) { + epSet.epSet.port[0] = port; + } + } else { + if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { + return NULL; + } + } + + char* key = getClusterKey(user, secretEncrypt, ip, port); + + SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + if (pInst == NULL) { + pInst = calloc(1, sizeof(struct SAppInstInfo)); + + pInst->mgmtEp = epSet; + pInst->pTransporter = openTransporter(user, secretEncrypt); + + taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES); + } + + return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst); +} + +int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { + pEpSet->version = 0; + + // init mgmt ip set + SEpSet *mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; + + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(firstEp, mgmtEpSet->fqdn[0], &(mgmtEpSet->port[0])); + mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(secondEp, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); + mgmtEpSet->numOfEps++; + } + + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + return 0; +} + +STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { + STscObj *pTscObj = createTscObj(user, auth, ip, port, pAppInfo); + if (NULL == pTscObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pTscObj; + } + + SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT); + if (pRequest == NULL) { + destroyTscObj(pTscObj); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SRequestMsgBody body = {0}; + buildConnectMsg(pRequest, &body); + + int64_t transporterId = 0; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyConnectMsg(&body); + + if (pRequest->code != TSDB_CODE_SUCCESS) { + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); + printf("failed to connect to server, reason: %s\n\n", errorMsg); + + destroyRequest(pRequest); + taos_close(pTscObj); + pTscObj = NULL; + } else { + tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter); + destroyRequest(pRequest); + } + + return pTscObj; +} + +static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; + pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); + if (pConnect == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + // TODO refactor full_name + char *db; // ugly code to move the space + + STscObj *pObj = pRequest->pTscObj; + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + pConnect->pid = htonl(appInfo.pid); + pConnect->startTime = htobe64(appInfo.startTime); + tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); + + pMsgBody->pData = pConnect; + return 0; +} + +static void destroyConnectMsg(SRequestMsgBody* pMsgBody) { + assert(pMsgBody != NULL); + tfree(pMsgBody->pData); +} + +int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) { + char *pMsg = rpcMallocCont(pBody->msgLen); + if (NULL == pMsg) { + tscError("0x%"PRIx64" msg:%s malloc failed", pBody->requestId, taosMsg[pBody->msgType]); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + memcpy(pMsg, pBody->pData, pBody->msgLen); + SRpcMsg rpcMsg = { + .msgType = pBody->msgType, + .pCont = pMsg, + .contLen = pBody->msgLen, + .ahandle = (void*) pBody->requestObjRefId, + .handle = NULL, + .code = 0 + }; + + rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId); + return TSDB_CODE_SUCCESS; +} + +void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + int64_t requestRefId = (int64_t)pMsg->ahandle; + + SRequestObj *pRequest = (SRequestObj *)taosAcquireRef(tscReqRef, requestRefId); + if (pRequest == NULL) { + rpcFreeCont(pMsg->pCont); + return; + } + + assert(pRequest->self == requestRefId); + pRequest->metric.rsp = taosGetTimestampMs(); + + pRequest->code = pMsg->code; + + STscObj *pTscObj = pRequest->pTscObj; + if (pEpSet) { + if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, pEpSet)) { + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, pEpSet); + } + } + + /* + * There is not response callback function for submit response. + * The actual inserted number of points is the first number. + */ + if (pMsg->code == TSDB_CODE_SUCCESS) { + tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); + if (handleRequestRspFp[pRequest->type]) { + pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen); + } + } else { + tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen); + } + + taosReleaseRef(tscReqRef, requestRefId); + rpcFreeCont(pMsg->pCont); + + sem_post(&pRequest->body.rspSem); +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c new file mode 100644 index 0000000000000000000000000000000000000000..62466a096df95950261ddd89d9caed7e3524870c --- /dev/null +++ b/source/client/src/clientMsgHandler.c @@ -0,0 +1,3212 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "clientInt.h" +#include "tmsgtype.h" +#include "trpc.h" +#include "tscLog.h" + +int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; +int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); + +#if 0 +void tscProcessActivityTimer(void *handle, void *tmrId); + +static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); + +static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static int32_t getWaitingTimeInterval(int32_t count) { + int32_t initial = 100; // 100 ms by default + if (count <= 1) { + return 0; + } + + return initial * ((2u)<<(count - 2)); +} + +static int32_t vgIdCompare(const void *lhs, const void *rhs) { + int32_t left = *(int32_t *)lhs; + int32_t right = *(int32_t *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} +static int32_t removeDupVgid(int32_t *src, int32_t sz) { + if (src == NULL || sz <= 0) { + return 0; + } + qsort(src, sz, sizeof(src[0]), vgIdCompare); + + int32_t ret = 1; + for (int i = 1; i < sz; i++) { + if (src[i] != src[i - 1]) { + src[ret++] = src[i]; + } + } + return ret; +} + +static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupMsg* pVgroupInfo) { + assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); + + // Issue the query to one of the vnode among a vgroup randomly. + // change the inUse property would not affect the isUse attribute of STableMeta + pEpSet->inUse = rand() % pVgroupInfo->numOfEps; + + // apply the FQDN string length check here + bool existed = false; + + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + + int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + if (len > 0) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + existed = true; + } + } + + assert(existed); +} + +static void tscDumpMgmtEpSet(SSqlObj *pSql) { + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + pSql->epSet = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); +} +static void tscEpSetHtons(SRpcEpSet *s) { + for (int32_t i = 0; i < s->numOfEps; i++) { + s->port[i] = htons(s->port[i]); + } +} + +bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { + return false; + } + + for (int32_t i = 0; i < s1->numOfEps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} + +void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { + // no need to update if equal + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginWrite(&pCorEpSet->version); + pCorEpSet->epSet = *pEpSet; + taosCorEndWrite(&pCorEpSet->version); +} + +static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { + if (pVgroupInfo == NULL) { return;} + int8_t inUse = pVgroupInfo->inUse; + pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); + pEpSet->port[i] = pVgroupInfo->ep[i].port; + } +} + +static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { + SSqlCmd *pCmd = &pSql->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { + return; + } + + int32_t vgId = -1; + if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { + vgId = extractSTableQueryVgroupId(pTableMetaInfo); + } else { + vgId = pTableMetaInfo->pTableMeta->vgId; + } + + assert(vgId > 0); + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); + assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); + + tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + vgroupInfo.inUse = pEpSet->inUse; + vgroupInfo.numOfEps = pEpSet->numOfEps; + for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { + tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + vgroupInfo.ep[i].port = pEpSet->port[i]; + } + + tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); + + // Update the local cached epSet info cached by SqlObj + int32_t inUse = pSql->epSet.inUse; + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + tscDebug("0x%"PRIx64" update the epSet in SqlObj, in use before:%d, after:%d", pSql->self, inUse, pSql->epSet.inUse); + +} + +int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { + assert(pTableMetaInfo != NULL); + + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + return vgId; +} + +void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { + STscObj *pObj = (STscObj *)param; + if (pObj == NULL) return; + + if (pObj != pObj->signature) { + tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); + return; + } + + SSqlObj *pSql = tres; + SSqlRes *pRes = &pSql->res; + + if (code == TSDB_CODE_SUCCESS) { + SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; + SRpcEpSet *epSet = &pRsp->epSet; + if (epSet->numOfEps > 0) { + tscEpSetHtons(epSet); + + //SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { + // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); + // for (int8_t i = 0; i < epSet->numOfEps; i++) { + // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); + // } + //} + //concurrency problem, update mgmt epset anyway + tscUpdateMgmtEpSet(pSql, epSet); + } + + pSql->pTscObj->connId = htonl(pRsp->connId); + + if (pRsp->killConnection) { + tscKillConnection(pObj); + return; + } else { + if (pRsp->queryId) { + tscKillQuery(pObj, htonl(pRsp->queryId)); + } + + if (pRsp->streamId) { + tscKillStream(pObj, htonl(pRsp->streamId)); + } + } + + int32_t total = htonl(pRsp->totalDnodes); + int32_t online = htonl(pRsp->onlineDnodes); + assert(online <= total); + + if (online < total) { + tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); + pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[0] = total; + pRes->length[1] = online; + } else { + tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[1] = 0; + if (pRes->length[0] == 0) { + pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node + } + } + + if (pObj->hbrid != 0) { + int32_t waitingDuring = tsShellActivityTimer * 500; + tscDebug("0x%"PRIx64" send heartbeat in %dms", pSql->self, waitingDuring); + + taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); + } else { + tscDebug("0x%"PRIx64" start to close tscObj:%p, not send heartbeat again", pSql->self, pObj); + } +} + +void tscProcessActivityTimer(void *handle, void *tmrId) { + int64_t rid = (int64_t) handle; + STscObj *pObj = taosAcquireRef(tscRefId, rid); + if (pObj == NULL) { + return; + } + + SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHB == NULL) { + taosReleaseRef(tscRefId, rid); + return; + } + + assert(pHB->self == pObj->hbrid); + + pHB->retry = 0; + int32_t code = tscBuildAndSendRequest(pHB, NULL); + taosReleaseRef(tscObjRef, pObj->hbrid); + + if (code != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); + } + + taosReleaseRef(tscRefId, rid); +} + +int tscSendMsgToServer(SSqlObj *pSql) { + STscObj* pObj = pSql->pTscObj; + SSqlCmd* pCmd = &pSql->cmd; + + char *pMsg = rpcMallocCont(pCmd->payloadLen); + if (NULL == pMsg) { + tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + // set the mgmt ip list + if (pSql->cmd.command >= TSDB_SQL_MGMT) { + tscDumpMgmtEpSet(pSql); + } + + memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); + + SRpcMsg rpcMsg = { + .msgType = pSql->cmd.msgType, + .pCont = pMsg, + .contLen = pSql->cmd.payloadLen, + .ahandle = (void*)pSql->self, + .handle = NULL, + .code = 0 + }; + + rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); + return TSDB_CODE_SUCCESS; +} + +// handle three situation +// 1. epset retry, only return last failure ep +// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn +// 3. other situation, no expected +void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + char* msgBuf = tscGetErrorMsgPayload(pCmd); + + if (pEpSet) { + sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); + } else if (pCmd->command >= TSDB_SQL_MGMT) { + SRpcEpSet tEpset; + + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + tEpset = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); + + sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); + } else { + sprintf(msgBuf, "%s", tstrerror(pRes->code)); + } +} + +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { + TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); + if (pSql == NULL) { + rpcFreeCont(rpcMsg->pCont); + return; + } + + assert(pSql->self == handle); + + STscObj *pObj = pSql->pTscObj; + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pSql->rpcRid = -1; + if (pObj->signature != pObj) { + tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); + + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { + tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", + pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); + + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + + if (pEpSet) { + if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { + if (pCmd->command < TSDB_SQL_MGMT) { + tscUpdateVgroupInfo(pSql, pEpSet); + } else { + tscUpdateMgmtEpSet(pSql, pEpSet); + } + } + } + + int32_t cmd = pCmd->command; + + // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema + if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + pSql->cmd.insertParam.schemaAttached = 1; + } + + // single table query error need to be handled here. + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || + rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { + + // 1. super table subquery + // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer + if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | + TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || + (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { + // do nothing in case of super table subquery + } else { + pSql->retry += 1; + tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); + + pSql->res.code = rpcMsg->code; // keep the previous error code + if (pSql->retry > pSql->maxRetry) { + tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); + } else { + // wait for a little bit moment and then retry + // todo do not sleep in rpc callback thread, add this process into queue to process + if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + int32_t duration = getWaitingTimeInterval(pSql->retry); + taosMsleep(duration); + } + + pSql->retryReason = rpcMsg->code; + rpcMsg->code = tscRenewTableMeta(pSql, 0); + // if there is an error occurring, proceed to the following error handling procedure. + if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + } + } + } + + pRes->rspLen = 0; + + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { + tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; + } + + if (pRes->code == TSDB_CODE_SUCCESS) { + tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); + pSql->retry = 0; + } + + if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { + assert(rpcMsg->msgType == pCmd->msgType + 1); + pRes->code = rpcMsg->code; + pRes->rspType = rpcMsg->msgType; + pRes->rspLen = rpcMsg->contLen; + + if (pRes->rspLen > 0 && rpcMsg->pCont) { + char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (tmp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + } else { + pRes->pRsp = tmp; + memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); + } + } else { + tfree(pRes->pRsp); + } + + /* + * There is not response callback function for submit response. + * The actual inserted number of points is the first number. + */ + if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) { + SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; + pMsg->code = htonl(pMsg->code); + pMsg->numOfRows = htonl(pMsg->numOfRows); + pMsg->affectedRows = htonl(pMsg->affectedRows); + pMsg->failedRows = htonl(pMsg->failedRows); + pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); + + pRes->numOfRows += pMsg->affectedRows; + tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], + tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); + } else { + tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); + } + } + + if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { + rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); + } + + bool shouldFree = tscShouldBeFreed(pSql); + if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + pRes->code = rpcMsg->code; + } + + rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; + if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) { + tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); + tscSetFqdnErrorMsg(pSql, pEpSet); + } + + (*pSql->fp)(pSql->param, pSql, rpcMsg->code); + } + + if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it + tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); + taosRemoveRef(tscObjRef, handle); + } + + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); +} + +int doBuildAndSendMsg(SSqlObj *pSql) { + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; + + if (pCmd->command == TSDB_SQL_SELECT || + pCmd->command == TSDB_SQL_FETCH || + pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_INSERT || + pCmd->command == TSDB_SQL_CONNECT || + pCmd->command == TSDB_SQL_HB || + pCmd->command == TSDB_SQL_RETRIEVE_FUNC || + pCmd->command == TSDB_SQL_STABLEVGROUP) { + pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); + } + + if (pRes->code != TSDB_CODE_SUCCESS) { + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; + } + + int32_t code = tscSendMsgToServer(pSql); + + // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. + if (code != TSDB_CODE_SUCCESS) { + pRes->code = code; + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + +int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + + SSqlCmd *pCmd = &pSql->cmd; + uint32_t type = 0; + + if (pQueryInfo == NULL) { + pQueryInfo = tscGetQueryInfo(pCmd); + } + + STableMetaInfo *pTableMetaInfo = NULL; + + if (pQueryInfo != NULL) { + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo != NULL) { + tNameExtractFullName(&pTableMetaInfo->name, name); + } + + type = pQueryInfo->type; + + // while numOfTables equals to 0, it must be Heartbeat + assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0); + } + + tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type); + if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL + if (pTableMetaInfo == NULL) { + pSql->res.code = TSDB_CODE_TSC_APP_ERROR; + return pSql->res.code; + } + } else if (pCmd->command >= TSDB_SQL_LOCAL) { + return (*tscProcessMsgRsp[pCmd->command])(pSql); + } + + return doBuildAndSendMsg(pSql); +} + +int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + + pRetrieveMsg->free = htons(pQueryInfo->type); + pRetrieveMsg->qId = htobe64(pSql->res.qId); + + // todo valid the vgroupId at the client side + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + pRetrieveMsg->header.vgId = htonl(vgId); + tscDebug("0x%"PRIx64" build fetch msg from vgId:%d, vgIndex:%d, qId:0x%" PRIx64, pSql->self, vgId, vgIndex, pSql->res.qId); + } else { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); + tscDebug("0x%"PRIx64" build fetch msg from only one vgroup, vgId:%d, qId:0x%" PRIx64, pSql->self, pTableMeta->vgId, + pSql->res.qId); + } + + pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); + pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH; + + pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + // pSql->cmd.payloadLen is set during copying data into payload + pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); + + return TSDB_CODE_SUCCESS; +} + +/* + * for table query, simply return the size <= 1k + */ +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { + const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; + + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); + int32_t srcColFilterSize = 0; + int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); + + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); + + int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; + + int32_t tableSerialize = 0; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + if (pTableMetaInfo->pVgroupTables != NULL) { + size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + + int32_t totalTables = 0; + for (int32_t i = 0; i < numOfGroups; ++i) { + SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); + totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); + } + + tableSerialize = totalTables * sizeof(STableIdInfo); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + srcColFilterSize = pCond->len; + } + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + srcColListSize += pCond->len; + } + + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + + exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; +} + +static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, + int32_t *succeed) { + TSKEY dfltKey = htobe64(pQueryMsg->window.skey); + + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { + + int32_t vgId = -1; + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + int32_t index = pTableMetaInfo->vgroupIndex; + assert(index >= 0); + + SVgroupMsg* pVgroupInfo = NULL; + if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { + assert(index < pTableMetaInfo->vgroupList->numOfVgroups); + pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; + } else { + tscError("0x%"PRIx64" No vgroup info found", pSql->self); + + *succeed = 0; + return pMsg; + } + + vgId = pVgroupInfo->vgId; + tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); + tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); + } else { + vgId = pTableMeta->vgId; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + } + + if (pSql->epSet.numOfEps > 0){ + pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; + } + pQueryMsg->head.vgId = htonl(vgId); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pTableMeta->id.tid); + pTableIdInfo->uid = htobe64(pTableMeta->id.uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); + + pQueryMsg->numOfTables = htonl(1); // set the number of tables + pMsg += sizeof(STableIdInfo); + } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo + int32_t index = pTableMetaInfo->vgroupIndex; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(index >= 0 && index < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); + + // set the vgroup info + tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); + pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); + + int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); + pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables + + tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, + pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); + + // serialize each table id info + for(int32_t i = 0; i < numOfTables; ++i) { + STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pItem->tid); + pTableIdInfo->uid = htobe64(pItem->uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); + pMsg += sizeof(STableIdInfo); + } + } + + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + + tscDebug("0x%"PRIx64" vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql->self, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); + return pMsg; +} + +// TODO refactor +static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { + // append the filter information after the basic column information + for (int32_t f = 0; f < numOfFilters; ++f) { + SColumnFilterInfo *pColFilter = &pColFilters[f]; + + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); + pFilterMsg->filterstr = htons(pColFilter->filterstr); + + (*pMsg) += sizeof(SColumnFilterInfo); + + if (pColFilter->filterstr) { + pFilterMsg->len = htobe64(pColFilter->len); + memcpy(*pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); + (*pMsg) += (pColFilter->len + 1); // append the additional filter binary info + } else { + pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); + pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); + } + + pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); + pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); + + if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { + tscError("invalid filter info"); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, char** pMsg, int64_t id, bool validateColumn) { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + // the queried table has been removed and a new table with the same name has already been created already + // return error msg + if (pExpr->uid != pTableMeta->id.uid) { + tscError("0x%"PRIx64" table has already been destroyed", id); + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + + if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { + tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + assert(pExpr->resColId < 0); + SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); + + SColIndex* pIndex = &pSqlExpr->colInfo; + + pIndex->colId = htons(pExpr->colInfo.colId); + pIndex->colIndex = htons(pExpr->colInfo.colIndex); + pIndex->flag = htons(pExpr->colInfo.flag); + pSqlExpr->uid = htobe64(pExpr->uid); + pSqlExpr->colType = htons(pExpr->colType); + pSqlExpr->colBytes = htons(pExpr->colBytes); + pSqlExpr->resType = htons(pExpr->resType); + pSqlExpr->resBytes = htons(pExpr->resBytes); + pSqlExpr->interBytes = htonl(pExpr->interBytes); + pSqlExpr->functionId = htons(pExpr->functionId); + pSqlExpr->numOfParams = htons(pExpr->numOfParams); + pSqlExpr->resColId = htons(pExpr->resColId); + pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters); + + (*pMsg) += sizeof(SSqlExpr); + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log + pSqlExpr->param[j].nType = htonl(pExpr->param[j].nType); + pSqlExpr->param[j].nLen = htonl(pExpr->param[j].nLen); + + if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { + memcpy((*pMsg), pExpr->param[j].pz, pExpr->param[j].nLen); + (*pMsg) += pExpr->param[j].nLen; + } else { + pSqlExpr->param[j].i64 = htobe64(pExpr->param[j].i64); + } + } + + serializeColFilterInfo(pExpr->flist.filterInfo, pExpr->flist.numOfFilters, pMsg); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = NULL; + STableMeta *pTableMeta = NULL; + STableMetaInfo *pTableMetaInfo = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + int32_t size = tscEstimateQueryMsgSize(pSql); + assert(size > 0); + + if (TSDB_CODE_SUCCESS != tscAllocPayloadFast(pCmd, size)) { + tscError("%p failed to malloc for query msg", pSql); + return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this + } + + pQueryInfo = tscGetQueryInfo(pCmd); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + pTableMeta = pTableMetaInfo->pTableMeta; + + SQueryAttr query = {{0}}; + tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); + query.vgId = pTableMeta->vgId; + + SArray* tableScanOperator = createTableScanPlan(&query); + SArray* queryOperator = createExecOperatorPlan(&query); + + SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; + tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); + + int32_t numOfTags = query.numOfTags; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); + + if (taosArrayGetSize(tableScanOperator) == 0) { + pQueryMsg->tableScanOperator = htonl(-1); + } else { + int32_t* tablescanOp = taosArrayGet(tableScanOperator, 0); + pQueryMsg->tableScanOperator = htonl(*tablescanOp); + } + + pQueryMsg->window.skey = htobe64(query.window.skey); + pQueryMsg->window.ekey = htobe64(query.window.ekey); + + pQueryMsg->order = htons(query.order.order); + pQueryMsg->orderColId = htons(query.order.orderColId); + pQueryMsg->fillType = htons(query.fillType); + pQueryMsg->limit = htobe64(query.limit.limit); + pQueryMsg->offset = htobe64(query.limit.offset); + pQueryMsg->numOfCols = htons(query.numOfCols); + + pQueryMsg->interval.interval = htobe64(query.interval.interval); + pQueryMsg->interval.sliding = htobe64(query.interval.sliding); + pQueryMsg->interval.offset = htobe64(query.interval.offset); + pQueryMsg->interval.intervalUnit = query.interval.intervalUnit; + pQueryMsg->interval.slidingUnit = query.interval.slidingUnit; + pQueryMsg->interval.offsetUnit = query.interval.offsetUnit; + + pQueryMsg->stableQuery = query.stableQuery; + pQueryMsg->topBotQuery = query.topBotQuery; + pQueryMsg->groupbyColumn = query.groupbyColumn; + pQueryMsg->hasTagResults = query.hasTagResults; + pQueryMsg->timeWindowInterpo = query.timeWindowInterpo; + pQueryMsg->queryBlockDist = query.queryBlockDist; + pQueryMsg->stabledev = query.stabledev; + pQueryMsg->tsCompQuery = query.tsCompQuery; + pQueryMsg->simpleAgg = query.simpleAgg; + pQueryMsg->pointInterpQuery = query.pointInterpQuery; + pQueryMsg->needReverseScan = query.needReverseScan; + pQueryMsg->stateWindow = query.stateWindow; + pQueryMsg->numOfTags = htonl(numOfTags); + pQueryMsg->sqlstrLen = htonl(sqlLen); + pQueryMsg->sw.gap = htobe64(query.sw.gap); + pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); + + pQueryMsg->secondStageOutput = htonl(query.numOfExpr2); + pQueryMsg->numOfOutput = htons((int16_t)query.numOfOutput); // this is the stage one output column number + + pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); + pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); + pQueryMsg->queryType = htonl(pQueryInfo->type); + pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); + + // set column list ids + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + char *pMsg = (char *)(pQueryMsg->tableCols) + numOfCols * sizeof(SColumnInfo); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfo *pCol = &query.tableCols[i]; + + pQueryMsg->tableCols[i].colId = htons(pCol->colId); + pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); + pQueryMsg->tableCols[i].type = htons(pCol->type); + //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); + pQueryMsg->tableCols[i].flist.numOfFilters = 0; + pQueryMsg->tableCols[i].flist.filterInfo = 0; + // append the filter information after the basic column information + //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->colCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } + } else { + pQueryMsg->colCondLen = 0; + } + + for (int32_t i = 0; i < query.numOfOutput; ++i) { + code = serializeSqlExpr(&query.pExpr1[i].base, pTableMetaInfo, &pMsg, pSql->self, true); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + + for (int32_t i = 0; i < query.numOfExpr2; ++i) { + code = serializeSqlExpr(&query.pExpr2[i].base, pTableMetaInfo, &pMsg, pSql->self, false); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + + int32_t succeed = 1; + + // serialize the table info (sid, uid, tags) + pMsg = doSerializeTableInfo(pQueryMsg, pSql, pTableMetaInfo, pMsg, &succeed); + if (succeed == 0) { + code = TSDB_CODE_TSC_APP_ERROR; + goto _end; + } + + SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; + if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { + pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); + pQueryMsg->orderType = htons(pGroupbyExpr->orderType); + + for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { + SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); + + *((int16_t *)pMsg) = htons(pCol->colId); + pMsg += sizeof(pCol->colId); + + *((int16_t *)pMsg) += htons(pCol->colIndex); + pMsg += sizeof(pCol->colIndex); + + *((int16_t *)pMsg) += htons(pCol->flag); + pMsg += sizeof(pCol->flag); + + memcpy(pMsg, pCol->name, tListLen(pCol->name)); + pMsg += tListLen(pCol->name); + } + } + + if (query.fillType != TSDB_FILL_NONE) { + for (int32_t i = 0; i < query.numOfOutput; ++i) { + *((int64_t *)pMsg) = htobe64(query.fillVal[i]); + pMsg += sizeof(query.fillVal[0]); + } + } + + if (query.numOfTags > 0 && query.tagColList != NULL) { + for (int32_t i = 0; i < query.numOfTags; ++i) { + SColumnInfo* pTag = &query.tagColList[i]; + + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + pTagCol->colId = htons(pTag->colId); + pTagCol->bytes = htons(pTag->bytes); + pTagCol->type = htons(pTag->type); + pTagCol->flist.numOfFilters = 0; + + pMsg += sizeof(SColumnInfo); + } + } + + // serialize tag column query condition + if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { + STagCond* pTagCond = &pQueryInfo->tagCond; + + SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->tagCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } + } else { + pQueryMsg->tagCondLen = 0; + } + + if (pQueryInfo->bufLen > 0) { + memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); + pMsg += pQueryInfo->bufLen; + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + strncpy(pMsg, pCond->cond, pCond->len); + pMsg += pCond->len; + } + + // compressed ts block + pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); + + if (pQueryInfo->tsBuf != NULL) { + // note: here used the index instead of actual vnode id. + int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; + code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + pMsg += pQueryMsg->tsBuf.tsLen; + + pQueryMsg->tsBuf.tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); + pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); + pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); + } else { + pQueryMsg->tsBuf.tsLen = 0; + pQueryMsg->tsBuf.tsNumOfBlocks = 0; + } + + int32_t numOfOperator = (int32_t) taosArrayGetSize(queryOperator); + pQueryMsg->numOfOperator = htonl(numOfOperator); + for(int32_t i = 0; i < numOfOperator; ++i) { + int32_t *operator = taosArrayGet(queryOperator, i); + *(int32_t*)pMsg = htonl(*operator); + + pMsg += sizeof(int32_t); + } + + // support only one udf + if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); + for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); + *(int8_t*) pMsg = pUdfInfo->resType; + pMsg += sizeof(pUdfInfo->resType); + + *(int16_t*) pMsg = htons(pUdfInfo->resBytes); + pMsg += sizeof(pUdfInfo->resBytes); + + STR_TO_VARSTR(pMsg, pUdfInfo->name); + + pMsg += varDataTLen(pMsg); + + *(int32_t*) pMsg = htonl(pUdfInfo->funcType); + pMsg += sizeof(pUdfInfo->funcType); + + *(int32_t*) pMsg = htonl(pUdfInfo->bufSize); + pMsg += sizeof(pUdfInfo->bufSize); + + pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen); + memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen); + + pMsg += pUdfInfo->contLen; + } + } else { + pQueryMsg->udfContentOffset = 0; + pQueryMsg->udfContentLen = 0; + } + + memcpy(pMsg, pSql->sqlstr, sqlLen); + pMsg += sqlLen; + + int32_t msgLen = (int32_t)(pMsg - pCmd->payload); + + tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); + pCmd->payloadLen = msgLen; + pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; + + pQueryMsg->head.contLen = htonl(msgLen); + assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); + + _end: + freeQueryAttr(&query); + taosArrayDestroy(tableScanOperator); + taosArrayDestroy(queryOperator); + return code; +} + +int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateDbMsg); + + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; + + SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; + +// assert(pCmd->numOfClause == 1); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); + assert(code == TSDB_CODE_SUCCESS); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_FUNCTION; + + pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateDnodeMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; + + SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); + strncpy(pCreate->ep, t0->z, t0->n); + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateAcctMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; + + SStrToken *pName = &pInfo->pMiscInfo->user.user; + SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; + + strncpy(pAlterMsg->user, pName->z, pName->n); + strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); + + SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; + + pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); + pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); + pAlterMsg->cfg.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); + pAlterMsg->cfg.maxStreams = htonl(pAcctOpt->maxStreams); + pAlterMsg->cfg.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); + pAlterMsg->cfg.maxStorage = htobe64(pAcctOpt->maxStorage); + pAlterMsg->cfg.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); + pAlterMsg->cfg.maxConnections = htonl(pAcctOpt->maxConnections); + + if (pAcctOpt->stat.n == 0) { + pAlterMsg->cfg.accessState = -1; + } else { + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + pAlterMsg->cfg.accessState = TSDB_VN_READ_ACCCESS; + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + pAlterMsg->cfg.accessState = TSDB_VN_WRITE_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + pAlterMsg->cfg.accessState = TSDB_VN_ALL_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + pAlterMsg->cfg.accessState = 0; + } + } + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateUserMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; + + SUserInfo *pUser = &pInfo->pMiscInfo->user; + strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); + pAlterMsg->flag = (int8_t)pUser->type; + + if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + pAlterMsg->privilege = (char)pCmd->count; + } else if (pUser->type == TSDB_ALTER_USER_PASSWD) { + strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); + } else { // create user password info + strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); + } + + if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + pCmd->msgType = TSDB_MSG_TYPE_ALTER_USER; + } else { + pCmd->msgType = TSDB_MSG_TYPE_CREATE_USER; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCfgDnodeMsg); + pCmd->msgType = TSDB_MSG_TYPE_CONFIG_DNODE; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SDropDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); + assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); + + pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + + pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_DROP_DB : TSDB_MSG_TYPE_DROP_TP; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->msgType = TSDB_MSG_TYPE_DROP_FUNCTION; + + pCmd->payloadLen = sizeof(SDropFuncMsg); + + return TSDB_CODE_SUCCESS; +} + + +int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCMDropTableMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); + + pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; + pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + pCmd->msgType = TSDB_MSG_TYPE_DROP_TABLE; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char dnodeEp[TSDB_EP_LEN] = {0}; + tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); + + pCmd->payloadLen = sizeof(SDropDnodeMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; + tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); + pCmd->msgType = TSDB_MSG_TYPE_DROP_DNODE; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char user[TSDB_USER_LEN] = {0}; + tstrncpy(user, pCmd->payload, TSDB_USER_LEN); + + pCmd->payloadLen = sizeof(SDropUserMsg); + pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_DROP_USER:TSDB_MSG_TYPE_DROP_ACCT; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropUserMsg *pDropMsg = (SDropUserMsg *)pCmd->payload; + tstrncpy(pDropMsg->user, user, tListLen(user)); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SUseDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_USE_DB; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SSyncDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_SYNC_DB; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_SHOW; + pCmd->payloadLen = sizeof(SShowMsg) + 100; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; + SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { + pShowMsg->type = pShowInfo->showType; + pShowMsg->payloadLen = 0; + pCmd->payloadLen = sizeof(SShowMsg); + + return TSDB_CODE_SUCCESS; + } + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + char *p = cloneCurrentDBName(pSql); + tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db)); + tfree(p); + } else { + tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); + } + + pShowMsg->type = pShowInfo->showType; + + if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { + SStrToken *pPattern = &pShowInfo->pattern; + if (pPattern->type > 0) { // only show tables support wildcard query + strncpy(pShowMsg->payload, pPattern->z, pPattern->n); + pShowMsg->payloadLen = htons(pPattern->n); + } + } else { + SStrToken *pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); + + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); + } + + pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SKillQueryMsg); + + switch (pCmd->command) { + case TSDB_SQL_KILL_QUERY: + pCmd->msgType = TSDB_MSG_TYPE_KILL_QUERY; + break; + case TSDB_SQL_KILL_CONNECTION: + pCmd->msgType = TSDB_MSG_TYPE_KILL_CONN; + break; + case TSDB_SQL_KILL_STREAM: + pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; + break; + } + return TSDB_CODE_SUCCESS; +} + +int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &(pSql->cmd); + int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); + + SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; + if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { + int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); + size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); + } else { + size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); + } + + if (pCreateTableInfo->pSelect != NULL) { + size += (pCreateTableInfo->pSelect->sqlstr.n + 1); + } + + return size + TSDB_EXTRA_PAYLOAD_SIZE; +} + +int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + int msgLen = 0; + int size = 0; + SSchema *pSchema; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + // Reallocate the payload size + size = tscEstimateCreateTableMsgLength(pSql, pInfo); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; + + SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); + char* pMsg = NULL; + + int8_t type = pInfo->pCreateTableInfo->type; + if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value + SArray* list = pInfo->pCreateTableInfo->childTableInfo; + + int32_t numOfTables = (int32_t) taosArrayGetSize(list); + pCreateTableMsg->numOfTables = htonl(numOfTables); + + pMsg = (char*) pCreateMsg; + for(int32_t i = 0; i < numOfTables; ++i) { + SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; + + pCreate->numOfColumns = htons(pCmd->numOfCols); + pCreate->numOfTags = htons(pCmd->count); + pMsg += sizeof(SCreateTableMsg); + + SCreatedTableInfo* p = taosArrayGet(list, i); + strcpy(pCreate->tableName, p->fullname); + pCreate->igExists = (p->igExist)? 1 : 0; + + // use dbinfo from table id without modifying current db info + pMsg = serializeTagData(&p->tagdata, pMsg); + + int32_t len = (int32_t)(pMsg - (char*) pCreate); + pCreate->len = htonl(len); + } + } else { // create (super) table + pCreateTableMsg->numOfTables = htonl(1); // only one table will be created + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); + assert(code == 0); + + SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; + + pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; + pCreateMsg->numOfColumns = htons(pCmd->numOfCols); + pCreateMsg->numOfTags = htons(pCmd->count); + + pCreateMsg->sqlLen = 0; + pMsg = (char *)pCreateMsg->schema; + + pSchema = (SSchema *)pCreateMsg->schema; + + for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + + pSchema->type = pField->type; + strcpy(pSchema->name, pField->name); + pSchema->bytes = htons(pField->bytes); + + pSchema++; + } + + pMsg = (char *)pSchema; + if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql + SSqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; + + strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); + pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); + pMsg += pQuerySql->sqlstr.n + 1; + } + } + + tscFieldInfoClear(&pQueryInfo->fieldsInfo); + + msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); + pCreateTableMsg->contLen = htonl(msgLen); + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + + assert(msgLen + minMsgSize() <= size); + return TSDB_CODE_SUCCESS; +} + +int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; +} + +int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + char *pMsg; + int msgLen = 0; + + SSqlCmd *pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; + int size = tscEstimateAlterTableMsgLength(pCmd); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; + + tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); + pAlterTableMsg->type = htons(pAlterInfo->type); + + pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); + SSchema *pSchema = pAlterTableMsg->schema; + for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + + pSchema->type = pField->type; + strcpy(pSchema->name, pField->name); + pSchema->bytes = htons(pField->bytes); + pSchema++; + } + + pMsg = (char *)pSchema; + pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); + if (pAlterInfo->tagData.dataLen > 0) { + memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); + } + pMsg += pAlterInfo->tagData.dataLen; + + msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); + + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_ALTER_TABLE; + + assert(msgLen + minMsgSize() <= size); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { + SSqlCmd* pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL; + + SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; + pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + assert(vgroupInfo.vgId > 0); + + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + return TSDB_CODE_SUCCESS; +} + +int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SAlterDbMsg); + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; + + SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; + pAlterDbMsg->dbType = -1; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); + + return TSDB_CODE_SUCCESS; +} +int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + if (pInfo->list == NULL || taosArrayGetSize(pInfo->list) <= 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + STscObj *pObj = pSql->pTscObj; + SSqlCmd *pCmd = &pSql->cmd; + SArray *pList = pInfo->list; + int32_t size = (int32_t)taosArrayGetSize(pList); + + int32_t *result = malloc(sizeof(int32_t) * size); + if (result == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < size; i++) { + tSqlExprItem* pSub = taosArrayGet(pList, i); + tVariant* pVar = &pSub->pNode->value; + if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { + result[i] = (int32_t)(pVar->i64); + } else { + free(result); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + int count = removeDupVgid(result, size); + pCmd->payloadLen = sizeof(SCompactMsg) + count * sizeof(int32_t); + pCmd->msgType = TSDB_MSG_TYPE_COMPACT_VNODE; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + free(result); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SCompactMsg *pCompactMsg = (SCompactMsg *)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + pthread_mutex_lock(&pObj->mutex); + tstrncpy(pCompactMsg->db, pObj->db, sizeof(pCompactMsg->db)); + pthread_mutex_unlock(&pObj->mutex); + } else { + tNameGetFullDbName(&pTableMetaInfo->name, pCompactMsg->db); + } + + pCompactMsg->numOfVgroup = htons(count); + for (int32_t i = 0; i < count; i++) { + pCompactMsg->vgid[i] = htons(result[i]); + } + free(result); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pCmd->payloadLen = sizeof(SRetrieveTableMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; + pRetrieveMsg->qId = htobe64(pSql->res.qId); + pRetrieveMsg->free = htons(pQueryInfo->type); + + return TSDB_CODE_SUCCESS; +} + +/* + * this function can only be called once. + * by using pRes->rspType to denote its status + * + * if pRes->rspType is 1, no more result + */ +static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pRes->code = TSDB_CODE_SUCCESS; + if (pRes->rspType == 0) { + pRes->numOfRows = numOfRes; + pRes->row = 0; + pRes->rspType = 1; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + tscSetResRawPtr(pRes, pQueryInfo); + } else { + tscResetForNextRetrieve(pRes); + } + + uint8_t code = pSql->res.code; + if (pSql->fp) { + if (code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); + } else { + tscAsyncResultOnError(pSql); + } + } + + return code; +} + +int tscProcessDescribeTableRsp(SSqlObj *pSql) { + SSqlCmd * pCmd = &pSql->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; + return tscLocalResultCommonBuilder(pSql, numOfRes); +} + +int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { + int32_t numOfRes = 1; + pSql->res.completed = true; + return tscLocalResultCommonBuilder(pSql, numOfRes); +} + +int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd* pCmd = &pSql->cmd; + + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscAsyncResultOnError(pSql); + return code; + } + + if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly. + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + return code; + } + + // global aggregation may be the upstream for parent query + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo->pQInfo == NULL) { + STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; + tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + + STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; + + SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(group, &tableKeyInfo); + taosArrayPush(tableGroupInfo.pGroupList, &group); + + tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute query processing", pSql->self, pSql->self); + pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE, pSql->self); + } + + uint64_t localQueryId = pSql->self; + qTableQuery(pQueryInfo->pQInfo, &localQueryId); + convertQueryResult(pRes, pQueryInfo, pSql->self, true); + + code = pRes->code; + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + } else { + tscAsyncResultOnError(pSql); + } + + return code; +} + +int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } + +int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + STscObj *pObj = pSql->pTscObj; + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_CONNECT; + pCmd->payloadLen = sizeof(SConnectMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; + + // TODO refactor full_name + char *db; // ugly code to move the space + + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); + tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); + + pConnect->pid = htonl(taosGetPId()); + taosGetCurrentAPPName(pConnect->appName, NULL); + + return TSDB_CODE_SUCCESS; +} + +/** + * multi table meta req pkg format: + * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... + * 4B + **/ +int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; + assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); + + tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, + pCmd->payloadLen); + + return pCmd->payloadLen; +} + +int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char* pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; + pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); + pMsg += sizeof(SSTableVgroupMsg); + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); + assert(code == TSDB_CODE_SUCCESS); + + pMsg += TSDB_TABLE_FNAME_LEN; + } + + pCmd->msgType = TSDB_MSG_TYPE_STABLE_VGROUP; + pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char *pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); + + SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; + pRetrieveFuncMsg->num = htonl(numOfFuncs); + + pMsg += sizeof(SRetrieveFuncMsg); + for(int32_t i = 0; i < numOfFuncs; ++i) { + SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i); + STR_TO_NET_VARSTR(pMsg, pUdf->name); + pMsg += varDataNetTLen(pMsg); + } + + pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE_FUNC; + pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + STscObj *pObj = pSql->pTscObj; + + pthread_mutex_lock(&pObj->mutex); + + int32_t numOfQueries = 2; + SSqlObj *tpSql = pObj->sqlList; + while (tpSql) { + tpSql = tpSql->next; + numOfQueries++; + } + + int32_t numOfStreams = 2; + SSqlStream *pStream = pObj->streamList; + while (pStream) { + pStream = pStream->next; + numOfStreams++; + } + + int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + pthread_mutex_unlock(&pObj->mutex); + tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + // TODO the expired hb and client can not be identified by server till now. + SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; + tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); + + pHeartbeat->numOfQueries = numOfQueries; + pHeartbeat->numOfStreams = numOfStreams; + + pHeartbeat->pid = htonl(taosGetPId()); + taosGetCurrentAPPName(pHeartbeat->appName, NULL); + + int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); + + pthread_mutex_unlock(&pObj->mutex); + + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); + + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); + + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && + (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, + pMetaMsg->tid, pMetaMsg->tableFname); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) { + tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { + tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { + pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); + } + + SSchema* pSchema = pMetaMsg->schema; + + int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; + for (int i = 0; i < numOfTotalCols; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema->colId = htons(pSchema->colId); + + if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + assert(i == 0); + } + + pSchema++; + } + + return TSDB_CODE_SUCCESS; +} + +// update the vgroupInfo if needed +static void doUpdateVgroupInfo(int32_t vgId, SVgroupMsg *pVgroupMsg) { + assert(vgId > 0); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); + + // vgroup info exists, compare with it + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { + vgroupInfo = createNewVgroupInfo(pVgroupMsg); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add/update new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } +} + +static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + // add or update the corresponding super table meta data info + int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + // The super tableMeta already exists, create it according to tableMeta and add it to hash map + if (updateSTable) { + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaMap, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); + + tfree(pSupTableMeta); + } + + CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); + taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); + tfree(cMeta); + } else { + uint32_t s = tscGetTableMetaSize(pTableMeta); + taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); + } +} + +int tscProcessTableMetaRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + assert(pTableMetaInfo->pTableMeta == NULL); + + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (pTableMeta == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); + tfree(pTableMeta); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); + + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); + if (pTableMeta->tableType != TSDB_SUPER_TABLE) { + doUpdateVgroupInfo(pTableMeta->vgId, &pMetaMsg->vgroup); + } + + tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, + pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, + pTableMeta->tableInfo.numOfTags); + + free(pTableMeta); + return TSDB_CODE_SUCCESS; +} + +static SArray* createVgroupIdListFromMsg(char* pMsg, SHashObj* pSet, char* name, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); + + SArray* vgroupIdList = taosArrayInit(pVgroupMsg->numOfVgroups, sizeof(int32_t)); + + if (pVgroupMsg->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup id list, no corresponding tables for stable:%s", id, name); + } else { + // just init, no need to lock + for (int32_t j = 0; j < pVgroupMsg->numOfVgroups; ++j) { + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } + + taosArrayPush(vgroupIdList, &vmsg->vgId); + + if (taosHashGet(pSet, &vmsg->vgId, sizeof(vmsg->vgId)) == NULL) { + taosHashPut(pSet, &vmsg->vgId, sizeof(vmsg->vgId), "", 0); + doUpdateVgroupInfo(vmsg->vgId, vmsg); + } + } + } + + return vgroupIdList; +} + +static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); + + size_t vgroupsz = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); + SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); + assert(pVgroupInfo != NULL); + + pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; + if (pVgroupInfo->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); + } else { + for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { + // just init, no need to lock + SVgroupMsg *pVgroup = &pVgroupInfo->vgroups[j]; + + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } + + pVgroup->numOfEps = vmsg->numOfEps; + pVgroup->vgId = vmsg->vgId; + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + pVgroup->epAddr[k].port = vmsg->epAddr[k].port; + tstrncpy(pVgroup->epAddr[k].fqdn, vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); +// pVgroup->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); + } + + doUpdateVgroupInfo(pVgroup->vgId, vmsg); + } + } + + return pVgroupInfo; +} + +int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + pFuncMsg->num = htonl(pFuncMsg->num); + assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); + + char* pMsg = pFuncMsg->content; + for(int32_t i = 0; i < pFuncMsg->num; ++i) { + SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; + + for(int32_t j = 0; j < pFuncMsg->num; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + if (strcmp(pUdfInfo->name, pFunc->name) != 0) { + continue; + } + + if (pUdfInfo->content) { + continue; + } + + pUdfInfo->resBytes = htons(pFunc->resBytes); + pUdfInfo->resType = pFunc->resType; + pUdfInfo->funcType = htonl(pFunc->funcType); + pUdfInfo->contLen = htonl(pFunc->len); + pUdfInfo->bufSize = htonl(pFunc->bufSize); + + pUdfInfo->content = malloc(pUdfInfo->contLen); + memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); + + pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; + } + } + + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + taosArrayDestroy(parQueryInfo->pUdfInfo); + + parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj. + pQueryInfo->pUdfInfo = NULL; + return TSDB_CODE_SUCCESS; +} + +int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { + char *rsp = pSql->res.pRsp; + + SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; + pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); + pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); + pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf); + + rsp += sizeof(SMultiTableMeta); + + SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(pParentSql == NULL) { + return pSql->res.code; + } + + SSqlCmd *pParentCmd = &pParentSql->cmd; + SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + + char* buf = NULL; + char* pMsg = pMultiMeta->meta; + + // decompresss the message payload + if (pMultiMeta->compressed) { + buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, + buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); + assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + + pMsg = buf; + } + + if (pParentCmd->pTableMetaMap == NULL) { + pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + } + + for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return code; + } + + bool freeMeta = false; + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); + tfree(pTableMeta); + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { + STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; + size_t keyLen = strnlen(pMetaMsg->tableFname, TSDB_TABLE_FNAME_LEN); + void* t = taosHashGet(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen); + assert(t == NULL); + + taosHashPut(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen, &p, sizeof(STableMetaVgroupInfo)); + } else { + freeMeta = true; + } + + // for each super table, only update meta information once + bool updateStableMeta = false; + if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { + updateStableMeta = true; + taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); + } + + // create the tableMeta and add it into the TableMeta map + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); + + // for each vgroup, only update the information once. + int64_t vgId = pMetaMsg->vgroup.vgId; + if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { + doUpdateVgroupInfo((int32_t) vgId, &pMetaMsg->vgroup); + taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); + } + + pMsg += pMetaMsg->contLen; + if (freeMeta) { + tfree(pTableMeta); + } + } + + for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { + char fname[TSDB_TABLE_FNAME_LEN] = {0}; + tstrncpy(fname, pMsg, TSDB_TABLE_FNAME_LEN); + size_t len = strnlen(fname, TSDB_TABLE_FNAME_LEN); + + pMsg += TSDB_TABLE_FNAME_LEN; + + STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, fname, len); + assert(p != NULL); + + int32_t size = 0; + if (p->vgroupIdList!= NULL) { + taosArrayDestroy(p->vgroupIdList); + } + + p->vgroupIdList = createVgroupIdListFromMsg(pMsg, pSet, fname, &size, pSql->self); + + int32_t numOfVgId = (int32_t) taosArrayGetSize(p->vgroupIdList); + int32_t s = sizeof(tFilePage) + numOfVgId * sizeof(int32_t); + + tFilePage* idList = calloc(1, s); + idList->num = numOfVgId; + memcpy(idList->data, TARRAY_GET_START(p->vgroupIdList), numOfVgId * sizeof(int32_t)); + + void* idListInst = taosCachePut(tscVgroupListBuf, fname, len, idList, s, 5000); + taosCacheRelease(tscVgroupListBuf, (void*) &idListInst, false); + + tfree(idList); + pMsg += size; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd); + if (pMultiMeta->numOfUdf > 0) { + assert(pQueryInfo->pUdfInfo != NULL); + } + + for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) { + SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; + + for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + if (strcmp(pUdfInfo->name, pFunc->name) != 0) { + continue; + } + + if (pUdfInfo->content) { + continue; + } + + pUdfInfo->resBytes = htons(pFunc->resBytes); + pUdfInfo->resType = pFunc->resType; + pUdfInfo->funcType = htonl(pFunc->funcType); + pUdfInfo->contLen = htonl(pFunc->len); + pUdfInfo->bufSize = htonl(pFunc->bufSize); + + pUdfInfo->content = malloc(pUdfInfo->contLen); + memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); + + pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; + } + } + + pSql->res.code = TSDB_CODE_SUCCESS; + pSql->res.numOfTotal = pMultiMeta->numOfTables; + tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); + + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return TSDB_CODE_SUCCESS; +} + +int tscProcessSTableVgroupRsp(SSqlObj *pSql) { + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + + SSqlRes* pRes = &pSql->res; + + // NOTE: the order of several table must be preserved. + SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; + pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); + char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); + + SSqlCmd* pCmd = &parent->cmd; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + char fName[TSDB_TABLE_FNAME_LEN] = {0}; + for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { + char* name = pMsg; + pMsg += TSDB_TABLE_FNAME_LEN; + + STableMetaInfo *pInfo = NULL; + for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { + STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); + memset(fName, 0, tListLen(fName)); + + tNameExtractFullName(&pInfo1->name, fName); + if (strcmp(name, fName) != 0) { + continue; + } + + pInfo = pInfo1; + break; + } + + if (!pInfo){ + continue; + } + int32_t size = 0; + pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); + pMsg += size; + } + + taosReleaseRef(tscObjRef, parent->self); + return pSql->res.code; +} + +int tscProcessShowRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg; + SShowRsp * pShow; + SSchema * pSchema; + + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + pShow = (SShowRsp *)pRes->pRsp; + pShow->qhandle = htobe64(pShow->qhandle); + pRes->qId = pShow->qhandle; + + tscResetForNextRetrieve(pRes); + pMetaMsg = &(pShow->tableMeta); + + pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); + + pSchema = pMetaMsg->schema; + pMetaMsg->tid = ntohs(pMetaMsg->tid); + for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema++; + } + + tfree(pTableMetaInfo->pTableMeta); + pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + + SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + SColumnIndex index = {0}; + pSchema = pMetaMsg->schema; + + uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; + for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { + index.columnIndex = i; + tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); + + TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); + SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); + + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, + pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); + } + + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + tscFieldInfoUpdateOffset(pQueryInfo); + return 0; +} + +static void createHbObj(STscObj* pObj) { + if (pObj->hbrid != 0) { + return; + } + + SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); + if (NULL == pSql) return; + + pSql->fp = tscProcessHeartBeatRsp; + + SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); + if (pQueryInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); + return; + } + + pQueryInfo->command = TSDB_SQL_HB; + + pSql->cmd.command = pQueryInfo->command; + if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); + return; + } + + pSql->param = pObj; + pSql->pTscObj = pObj; + pSql->signature = pSql; + + registerSqlObj(pSql); + tscDebug("0x%"PRIx64" HB is allocated, pObj:%p", pSql->self, pObj); + + pObj->hbrid = pSql->self; +} + +int tscProcessUseDbRsp(SSqlObj *pSql) { + STscObj * pObj = pSql->pTscObj; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + + pthread_mutex_lock(&pObj->mutex); + int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + pthread_mutex_unlock(&pObj->mutex); + + return ret; +} + +//todo only invalid the buffered data that belongs to dropped databases +int tscProcessDropDbRsp(SSqlObj *pSql) { + //TODO LOCK DB WHEN MODIFY IT + //pSql->pTscObj->db[0] = 0; + + taosHashClear(tscTableMetaMap); + taosHashClear(tscVgroupMap); + taosCacheEmpty(tscVgroupListBuf); + return 0; +} + +int tscProcessDropTableRsp(SSqlObj *pSql) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); + tfree(pTableMetaInfo->pTableMeta); + return 0; +} + +int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + tscDebug("0x%"PRIx64" remove tableMeta in hashMap after alter-table: %s", pSql->self, name); + + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tfree(pTableMetaInfo->pTableMeta); + + if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta + if (pSql->res.pRsp == NULL) { + tscDebug("0x%"PRIx64" unexpected resp from mnode, super table: %s failed to update super table meta ", pSql->self, name); + return 0; + } + return tscProcessTableMetaRsp(pSql); + } + + return 0; +} + +int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { + UNUSED(pSql); + return 0; +} +int tscProcessCompactRsp(SSqlObj *pSql) { + UNUSED(pSql); + return TSDB_CODE_SUCCESS; +} + +int tscProcessShowCreateRsp(SSqlObj *pSql) { + return tscLocalResultCommonBuilder(pSql, 1); +} + +int tscProcessQueryRsp(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + + SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp; + pQueryAttr->qId = htobe64(pQueryAttr->qId); + + pRes->qId = pQueryAttr->qId; + pRes->data = NULL; + + tscResetForNextRetrieve(pRes); + tscDebug("0x%"PRIx64" query rsp received, qId:0x%"PRIx64, pSql->self, pRes->qId); + return 0; +} + +static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { + int32_t decompLen = 0; + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + int32_t *compSizes; + char *pData = *data; + compSizes = (int32_t *)(pData + compLen); + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); + int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); + char *outputBuf = tcalloc(pRes->numOfRows, (pField->bytes + offset)); + + char *p = outputBuf; + int32_t bufOffset; + for (int32_t i = 0; i < numOfCols; ++i) { + SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); + bufOffset = pInfo->field.bytes * pRes->numOfRows; + + int32_t flen = (*(tDataTypes[pInfo->field.type].decompFunc))(pData, htonl(compSizes[i]), pRes->numOfRows, p, bufOffset, + compressed, NULL, 0); + + p += flen; + decompLen +=flen; + pData += htonl(compSizes[i]); + } + + /* Resize rsp as decompressed data will occupy more space */ + pRes->rspLen = pRes->rspLen - (compLen + numOfCols * sizeof(int32_t)) + decompLen; + char *new_rsp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (new_rsp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return; + } else { + pRes->pRsp = new_rsp; + *data = ((SRetrieveTableRsp *)pRes->pRsp)->data; + pData = *data + compLen + numOfCols * sizeof(int32_t); + } + + tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", + pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); + + int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; + memmove(*data + decompLen, pData, tailLen); + memmove(*data, outputBuf, decompLen); + + tfree(outputBuf); +} + +int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); + + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; + if (pRetrieve == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; + } + + pRes->numOfRows = htonl(pRetrieve->numOfRows); + pRes->precision = htons(pRetrieve->precision); + pRes->offset = htobe64(pRetrieve->offset); + pRes->useconds = htobe64(pRetrieve->useconds); + pRes->completed = (pRetrieve->completed == 1); + pRes->data = pRetrieve->data; + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + //Decompress col data if compressed from server + if (pRetrieve->compressed) { + int32_t compLen = htonl(pRetrieve->compLen); + decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); + } + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if ((pCmd->command == TSDB_SQL_RETRIEVE) || + ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || + (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { + tscSetResRawPtr(pRes, pQueryInfo); + } + + if (pSql->pSubscription != NULL) { + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); + int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); + + char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; + + int32_t numOfTables = htonl(*(int32_t*)p); + p += sizeof(int32_t); + for (int i = 0; i < numOfTables; i++) { + int64_t uid = htobe64(*(int64_t*)p); + p += sizeof(int64_t); + p += sizeof(int32_t); // skip tid + TSKEY key = htobe64(*(TSKEY*)p); + p += sizeof(TSKEY); + tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); + } + } + + pRes->row = 0; + tscDebug("0x%"PRIx64" numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:0x%"PRIx64, pSql->self, pRes->numOfRows, pRes->offset, + pRes->completed, pRes->qId); + + return 0; +} + +void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); + +static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_META; + + tscAddQueryInfo(&pNew->cmd); + + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); + assert(pNewQueryInfo->numOfTables == 1); + + tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); + + registerSqlObj(pNew); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, + pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); + pSql->metaRid = pNew->self; + + { + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; + int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + pInfoMsg->createFlag = htons(autocreate? 1 : 0); + char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); + + // tag data exists + if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) { + pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg); + } + + pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); + pNew->cmd.msgType = TSDB_MSG_TYPE_TABLE_META; + } + + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_MULTI_META; + + int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); + int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); + int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0; + + int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; + pInfo->metaClone = metaClone? 1:0; + pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); + pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); + pInfo->numOfUdfs = htonl(numOfUdf); + + char* start = pInfo->tableNames; + int32_t len = 0; + for(int32_t i = 0; i < numOfTable; ++i) { + char* name = taosArrayGetP(pNameList, i); + if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfVgroupList; ++i) { + char* name = taosArrayGetP(pVgroupNameList, i); + if (i < numOfVgroupList - 1 || numOfUdf > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfUdf; ++i) { + SUdfInfo * u = taosArrayGet(pUdfList, i); + if (i < numOfUdf - 1) { + len = sprintf(start, "%s,", u->name); + } else { + len = sprintf(start, "%s", u->name); + } + + start += len; + } + + pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); + pNew->cmd.msgType = TSDB_MSG_TYPE_TABLES_META; + + registerSqlObj(pNew); + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, + pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen); + + pNew->fp = fp; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" metaRid from 0x%" PRIx64 " to 0x%" PRIx64 , pSql->self, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { + assert(tIsValidName(&pTableMetaInfo->name)); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + size_t len = strlen(name); + // just make runtime happy + if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { + memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); + } + + if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity)) { + tfree(pTableMetaInfo->pTableMeta); + } + + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); + + if (pMeta && pMeta->id.uid > 0) { + // in case of child table, here only get the + if (pMeta->tableType == TSDB_CHILD_TABLE) { + int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); + pSql->pBuf = (void *)(pSTMeta); + if (code != TSDB_CODE_SUCCESS) { + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); + } + } + + tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); + return TSDB_CODE_SUCCESS; + } + + if (onlyLocal) { + return TSDB_CODE_TSC_NO_META_CACHED; + } + + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); +} + +int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, false, false); +} + +int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists, onlyLocal); +} + +int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC; + + if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) { + tscError("%p malloc failed for new queryinfo", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); + + pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); + for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { + SUdfInfo info = {0}; + SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i); + info = *p1; + info.name = strdup(p1->name); + taosArrayPush(pNewQueryInfo->pUdfInfo, &info); + } + + pNew->cmd.active = pNewQueryInfo; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { + tscError("%p malloc failed for payload to get table meta", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew); + registerSqlObj(pNew); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + + tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +static void freeElem(void* p) { + tfree(*(char**)p); +} + +/** + * retrieve table meta from mnode, and then update the local table meta hashmap. + * @param pSql sql object + * @param tableIndex table index + * @return status code + */ +int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { + SSqlCmd* pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); + if (code != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pTableMeta) { + tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); + } + + + // remove stored tableMeta info in hash table + tscResetSqlCmd(pCmd, true, pSql->self); + + SArray* pNameList = taosArrayInit(1, POINTER_BYTES); + SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); + + char* n = strdup(name); + taosArrayPush(pNameList, &n); + code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); + taosArrayDestroyEx(pNameList, freeElem); + taosArrayDestroyEx(vgroupList, freeElem); + + return code; +} + +static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (pTableMetaInfo->vgroupList == NULL) { + return false; + } + } + + // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore + return true; +} + +int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (allVgroupInfoRetrieved(pQueryInfo)) { + return TSDB_CODE_SUCCESS; + } + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + + pNew->cmd.command = TSDB_SQL_STABLEVGROUP; + + // TODO TEST IT + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); + if (pNewQueryInfo == NULL) { + tscFreeSqlObj(pNew); + return code; + } + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); + STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); + tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); + } + + if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pNew); + return code; + } + + pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + registerSqlObj(pNew); + + tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self); + + pSql->svgroupRid = pNew->self; + tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; + } + + return code; +} + +#endif + +int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + STscObj *pTscObj = pRequest->pTscObj; + + SConnectRsp *pConnect = (SConnectRsp *)pMsg; + pConnect->acctId = htonl(pConnect->acctId); + pConnect->connId = htonl(pConnect->connId); + pConnect->clusterId = htonl(pConnect->clusterId); + + // TODO refactor + pthread_mutex_lock(&pTscObj->mutex); + char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; + int32_t len = sprintf(temp, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db); + + assert(len <= sizeof(pTscObj->db)); + tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db)); + pthread_mutex_unlock(&pTscObj->mutex); + + assert(pConnect->epSet.numOfEps > 0); + if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet); + } + + for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id); + } + + pTscObj->connId = pConnect->connId; + + // update the appInstInfo + pTscObj->pAppInfo->clusterId = pConnect->clusterId; + atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + + tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); + // createHbObj(pTscObj); + + // launch a timer to send heartbeat to maintain the connection and send status to mnode + // taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer); + return 0; +} + +void initMsgHandleFp() { +#if 0 + tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; + tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; + tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; + + tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; + tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; + + tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; + tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; + + tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; + tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; + tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; + tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; + tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; + tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; + tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; + tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; + tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; + + + tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; + + tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; + tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; + + tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; + + tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; + tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; + + tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; + tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; + + tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; + + tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; + tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; + tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; +#endif + +// buildRequestMsgFp[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; +} \ No newline at end of file diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c new file mode 100644 index 0000000000000000000000000000000000000000..ba801358502b0a48a102afc338c412f7a00aa657 --- /dev/null +++ b/source/client/src/clientmain.c @@ -0,0 +1,85 @@ +#include "clientInt.h" +#include "trpc.h" +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "tscLog.h" +#include "tsched.h" +#include "ttime.h" +#include "ttimezone.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +static int32_t sentinel = TSC_VAR_NOT_RELEASE; +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; + +extern int32_t tscInitRes; + +int taos_options(TSDB_OPTION option, const void *arg, ...) { + static int32_t lock = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscInfo("haven't acquire lock after spin %d times.", i); + sched_yield(); + } + } + + int ret = taos_options_imp(option, (const char*)arg); + + atomic_store_32(&lock, 0); + return ret; +} + +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); + + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; + } + + int32_t id = tscReqRef; + tscReqRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscConnRef; + tscConnRef = -1; + taosCloseRef(id); + + rpcCleanup(); + taosCloseLog(); +} + +void taos_close(TAOS* taos) { + if (taos == NULL) { + return; + } + + STscObj *pTscObj = (STscObj *)taos; + tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + + taosRemoveRef(tscConnRef, pTscObj->id); +} + +const char *taos_errstr(TAOS_RES *res) { + +} + +void taos_free_result(TAOS_RES *res) { + +} \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c new file mode 100644 index 0000000000000000000000000000000000000000..43d73bf3db85732037e597ab62bf0b0a2830c1dd --- /dev/null +++ b/source/client/src/tscEnv.c @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "tscLog.h" +#include "tsched.h" +#include "ttime.h" +#include "trpc.h" +#include "ttimezone.h" +#include "clientInt.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +SAppInfo appInfo; +int32_t tscReqRef = -1; +int32_t tscConnRef = -1; +void *tscQhandle = NULL; + +int32_t tsNumOfThreads = 1; +volatile int32_t tscInitRes = 0; + +static void registerRequest(SRequestObj* pRequest) { + STscObj *pTscObj = (STscObj *)taosAcquireRef(tscConnRef, pRequest->pTscObj->id); + assert(pTscObj != NULL); + + // connection has been released already, abort creating request. + pRequest->self = taosAddRef(tscReqRef, pRequest); + + int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); + + if (pTscObj->pAppInfo) { + SInstanceActivity *pActivity = &pTscObj->pAppInfo->summary; + + int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d", pRequest->self, + pRequest->pTscObj->id, num, currentInst, total); + } +} + +static void deregisterRequest(SRequestObj* pRequest) { + assert(pRequest != NULL); + + STscObj* pTscObj = pRequest->pTscObj; + SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; + + int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + + tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + taosReleaseRef(tscConnRef, pTscObj->id); +} + +static void tscInitLogFile() { + taosReadGlobalLogCfg(); + if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { + printf("failed to create log dir:%s\n", tsLogDir); + } + + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + +void closeTransporter(STscObj* pTscObj) { + if (pTscObj == NULL || pTscObj->pTransporter == NULL) { + return; + } + + tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id); + rpcClose(pTscObj->pTransporter); + pTscObj->pTransporter = NULL; +} + +// TODO refactor +void* openTransporter(const char *user, const char *auth) { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "TSC"; + rpcInit.numOfThreads = tsNumOfThreads; + rpcInit.cfp = processMsgFromServer; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)user; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; +// rpcInit.spi = 1; + rpcInit.secret = (char *)auth; + + void* pDnodeConn = rpcOpen(&rpcInit); + if (pDnodeConn == NULL) { + tscError("failed to init connection to server"); + return NULL; + } + + return pDnodeConn; +} + +void destroyTscObj(void *pObj) { + STscObj *pTscObj = pObj; + + atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + + closeTransporter(pTscObj); + pthread_mutex_destroy(&pTscObj->mutex); + tfree(pTscObj); +} + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo) { + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); + if (NULL == pObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + pObj->pAppInfo = pAppInfo; + if (pAppInfo != NULL) { + pObj->pTransporter = pAppInfo->pTransporter; + } + + tstrncpy(pObj->user, user, sizeof(pObj->user)); + memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); + + pthread_mutex_init(&pObj->mutex, NULL); + pObj->id = taosAddRef(tscConnRef, pObj); + + tscDebug("connObj created, 0x%"PRIx64, pObj->id); + return pObj; +} + +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { + assert(pObj != NULL); + + SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + if (NULL == pRequest) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + // TODO generated request uuid + pRequest->requestId = 0; + + pRequest->metric.start = taosGetTimestampMs(); + + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; + pRequest->body.param = param; + tsem_init(&pRequest->body.rspSem, 0, 0); + + registerRequest(pRequest); + return pRequest; +} + +static void doDestroyRequest(void* p) { + assert(p != NULL); + SRequestObj* pRequest = (SRequestObj*)p; + + assert(RID_VALID(pRequest->self)); + + tfree(pRequest->msgBuf); + tfree(pRequest->sqlstr); + tfree(pRequest->pInfo); + + deregisterRequest(pRequest); + tfree(pRequest); +} + +void destroyRequest(SRequestObj* pRequest) { + if (pRequest == NULL) { + return; + } + + taosReleaseRef(tscReqRef, pRequest->self); +} + +void taos_init_imp(void) { + // In the APIs of other program language, taos_cleanup is not available yet. + // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. + atexit(taos_cleanup); + + errno = TSDB_CODE_SUCCESS; + srand(taosGetTimestampSec()); + + deltaToUtcInitOnce(); + taosInitGlobalCfg(); + taosReadCfgFromFile(); + + tscInitLogFile(); + if (taosCheckAndPrintCfg()) { + tscInitRes = -1; + return; + } + + taosInitNotes(); + initMsgHandleFp(); + + rpcInit(); + + tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + + taosSetCoreDump(true); + + double factor = 4.0; + int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + + int32_t queueSize = tsMaxConnections * 2; + tscQhandle = taosInitScheduler(queueSize, numOfThreads, "tsc"); + if (NULL == tscQhandle) { + tscError("failed to init task queue"); + tscInitRes = -1; + return; + } + + tscDebug("client task queue is initialized, numOfThreads: %d", numOfThreads); + tscConnRef = taosOpenRef(200, destroyTscObj); + tscReqRef = taosOpenRef(40960, doDestroyRequest); + + taosGetAppName(appInfo.appName, NULL); + appInfo.pid = taosGetPId(); + appInfo.startTime = taosGetTimestampMs(); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + tscDebug("client is initialized successfully"); +} + +int taos_options_imp(TSDB_OPTION option, const char *str) { + SGlobalCfg *cfg = NULL; + + switch (option) { + case TSDB_OPTION_CONFIGDIR: + cfg = taosGetConfigOption("configDir"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(configDir, str, TSDB_FILENAME_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set config file directory:%s", str); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + case TSDB_OPTION_SHELL_ACTIVITY_TIMER: + cfg = taosGetConfigOption("shellActivityTimer"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tsShellActivityTimer = atoi(str); + if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; + if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + } + break; + + case TSDB_OPTION_LOCALE: { // set locale + cfg = taosGetConfigOption("locale"); + assert(cfg != NULL); + + size_t len = strlen(str); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("Invalid locale:%s, use default", str); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + char sep = '.'; + + if (strlen(tsLocale) == 0) { // locale does not set yet + char* defaultLocale = setlocale(LC_CTYPE, ""); + + // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. + // The launch of current system will abort soon. + if (defaultLocale == NULL) { + tscError("failed to get default locale, please set the correct locale in current OS"); + return -1; + } + + tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + } + + // set the user specified locale + char *locale = setlocale(LC_CTYPE, str); + + if (locale != NULL) { // failed to set the user specified locale + tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { // set the user specified locale failed, use default LC_CTYPE as current locale + locale = setlocale(LC_CTYPE, tsLocale); + tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); + } + + tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + + char *charset = strrchr(tsLocale, sep); + if (charset != NULL) { + charset += 1; + + charset = taosCharsetReplace(charset); + + if (taosValidateEncodec(charset)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset set:%s", charset); + } else { + tscInfo("charset changed from %s to %s", tsCharset, charset); + } + + tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + + } else { + tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); + } + + free(charset); + } else { // it may be windows system + tscInfo("charset remains:%s", tsCharset); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + } + + case TSDB_OPTION_CHARSET: { + /* set charset will override the value of charset, assigned during system locale changed */ + cfg = taosGetConfigOption("charset"); + assert(cfg != NULL); + + size_t len = strlen(str); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("failed to set charset:%s", str); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + if (taosValidateEncodec(str)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset is set:%s", str); + } else { + tscInfo("charset changed from %s to %s", tsCharset, str); + } + + tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { + tscInfo("charset:%s not valid", str); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + + break; + } + + case TSDB_OPTION_TIMEZONE: + cfg = taosGetConfigOption("timezone"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); + tsSetTimeZone(); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + default: + // TODO return the correct error code to client in the format for taos_errstr() + tscError("Invalid option %d", option); + return -1; + } + + return 0; +} + +#if 0 +#include "cJSON.h" +static setConfRet taos_set_config_imp(const char *config){ + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + static bool setConfFlag = false; + if (setConfFlag) { + ret.retCode = SET_CONF_RET_ERR_ONLY_ONCE; + strcpy(ret.retMsg, "configuration can only set once"); + return ret; + } + taosInitGlobalCfg(); + cJSON *root = cJSON_Parse(config); + if (root == NULL){ + ret.retCode = SET_CONF_RET_ERR_JSON_PARSE; + strcpy(ret.retMsg, "parse json error"); + return ret; + } + + int size = cJSON_GetArraySize(root); + if(!cJSON_IsObject(root) || size == 0) { + ret.retCode = SET_CONF_RET_ERR_JSON_INVALID; + strcpy(ret.retMsg, "json content is invalid, must be not empty object"); + return ret; + } + + if(size >= 1000) { + ret.retCode = SET_CONF_RET_ERR_TOO_LONG; + strcpy(ret.retMsg, "json object size is too long"); + return ret; + } + + for(int i = 0; i < size; i++){ + cJSON *item = cJSON_GetArrayItem(root, i); + if(!item) { + ret.retCode = SET_CONF_RET_ERR_INNER; + strcpy(ret.retMsg, "inner error"); + return ret; + } + if(!taosReadConfigOption(item->string, item->valuestring, NULL, NULL, TAOS_CFG_CSTATUS_OPTION, TSDB_CFG_CTYPE_B_CLIENT)){ + ret.retCode = SET_CONF_RET_ERR_PART; + if (strlen(ret.retMsg) == 0){ + snprintf(ret.retMsg, RET_MSG_LENGTH, "part error|%s", item->string); + }else{ + int tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + size_t leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, "|", leftSize); + tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, item->string, leftSize); + } + } + } + cJSON_Delete(root); + setConfFlag = true; + return ret; +} + +setConfRet taos_set_config(const char *config){ + pthread_mutex_lock(&setConfMutex); + setConfRet ret = taos_set_config_imp(config); + pthread_mutex_unlock(&setConfMutex); + return ret; +} +#endif \ No newline at end of file diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3f2ad88bb6d8fc35e17ec76dd18539655a607fd --- /dev/null +++ b/source/client/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build parser unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(clientTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + clientTest + PUBLIC os util common transport gtest taos +) + +TARGET_INCLUDE_DIRECTORIES( + clientTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc" +) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8da4caf7eb53c58dd29a4d4b605ff9b2e1089ee9 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "tglobal.h" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "taos.h" + +namespace { +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, driverInit_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + taos_close(pConn); +} \ No newline at end of file diff --git a/source/common/inc/commonInt.h b/source/common/inc/commonInt.h index c1baa2a453d61d04c92c16c60c9e20ea9de8d1d7..448600e22e0ab525dcd143d485603cee589b9a4b 100644 --- a/source/common/inc/commonInt.h +++ b/source/common/inc/commonInt.h @@ -24,12 +24,11 @@ extern "C" { #include "tlog.h" extern int32_t cDebugFlag; -extern int8_t tscEmbedded; -#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) diff --git a/source/common/src/tep.c b/source/common/src/tep.c new file mode 100644 index 0000000000000000000000000000000000000000..ef47a0e0c2a5abc0a21692bd88fb1030ecf07de8 --- /dev/null +++ b/source/common/src/tep.c @@ -0,0 +1,41 @@ +#include "tep.h" +#include "tglobal.h" +#include "tlockfree.h" + +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { + *port = 0; + strcpy(fqdn, ep); + + char *temp = strchr(fqdn, ':'); + if (temp) { + *temp = 0; + *port = atoi(temp+1); + } + + if (*port == 0) { + *port = tsServerPort; + return -1; + } + + return 0; +} + +bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { + return false; + } + + for (int32_t i = 0; i < s1->numOfEps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} + +void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet) { + taosCorBeginWrite(&pEpSet->version); + pEpSet->epSet = *pNewEpSet; + taosCorEndWrite(&pEpSet->version); +} + diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 60e841539231cddb87c6a66dac3608115ca62a9a..dfbbe33b59850ccc1bb29dffbbcaf4027c434a8b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -25,6 +25,7 @@ #include "tutil.h" #include "ttimezone.h" #include "tlocale.h" +#include "tep.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -46,7 +47,7 @@ int64_t tsDnodeStartTime = 0; // common int32_t tsRpcTimer = 300; int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 0; //disable this, means query, show command use udp protocol as default +int32_t tsRpcForceTcp = 1; //disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; int32_t tsShellActivityTimer = 3; // second @@ -156,22 +157,13 @@ int8_t tsEnableBalance = 1; int8_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds int32_t tsOfflineThreshold = 86400 * 10; // seconds of 10 days -int32_t tsMnodeEqualVnodeNum = 4; int8_t tsEnableFlowCtrl = 1; int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; // restful -int8_t tsEnableHttpModule = 1; int32_t tsRestRowLimit = 10240; -uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041] -int32_t tsHttpCacheSessions = 1000; -int32_t tsHttpSessionExpire = 36000; -int32_t tsHttpMaxThreads = 2; -int8_t tsHttpEnableCompress = 1; -int8_t tsHttpEnableRecordSql = 0; int8_t tsTelegrafUseFieldNum = 0; -int8_t tsHttpDbNameMandatory = 0; // mqtt int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default @@ -194,7 +186,6 @@ int8_t tsEnableStream = 1; // internal int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; -int8_t tscEmbedded = 0; char tsVnodeDir[PATH_MAX] = {0}; char tsDnodeDir[PATH_MAX] = {0}; char tsMnodeDir[PATH_MAX] = {0}; @@ -257,7 +248,6 @@ void taosSetAllDebugFlag() { sdbDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; - cDebugFlag = debugFlag; jniDebugFlag = debugFlag; odbcDebugFlag = debugFlag; httpDebugFlag = debugFlag; @@ -331,7 +321,7 @@ int32_t taosCfgDynamicOptions(char *msg) { if (strncasecmp(option, "resetlog", 8) == 0) { taosResetLog(); - taosPrintGlobalCfg(); + taosPrintCfg(); return 0; } @@ -395,7 +385,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "secondEp"; cfg.ptr = tsSecond; @@ -405,7 +395,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fqdn"; cfg.ptr = tsLocalFqdn; @@ -415,7 +405,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FQDN_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // port cfg.option = "serverPort"; @@ -426,7 +416,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65056; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // directory cfg.option = "configDir"; @@ -437,7 +427,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "logDir"; cfg.ptr = tsLogDir; @@ -447,7 +437,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "scriptDir"; cfg.ptr = tsScriptDir; @@ -457,7 +447,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dataDir"; cfg.ptr = tsDataDir; @@ -467,7 +457,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "arbitrator"; cfg.ptr = tsArbitrator; @@ -477,7 +467,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // dnode configs cfg.option = "numOfThreadsPerCore"; @@ -488,7 +478,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "numOfCommitThreads"; cfg.ptr = &tsNumOfCommitThreads; @@ -498,7 +488,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "ratioOfQueryCores"; cfg.ptr = &tsRatioOfQueryCores; @@ -508,7 +498,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxNumOfDistinctRes"; cfg.ptr = &tsMaxNumOfDistinctResults; @@ -518,7 +508,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000*10000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "numOfMnodes"; cfg.ptr = &tsNumOfMnodes; @@ -528,7 +518,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 3; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "vnodeBak"; cfg.ptr = &tsEnableVnodeBak; @@ -538,7 +528,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; @@ -548,7 +538,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "balance"; cfg.ptr = &tsEnableBalance; @@ -558,7 +548,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "balanceInterval"; cfg.ptr = &tsBalanceInterval; @@ -568,7 +558,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 30000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // 0-any; 1-mnode; 2-vnode cfg.option = "role"; @@ -579,7 +569,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // timer cfg.option = "maxTmrCtrl"; @@ -590,7 +580,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2048; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monitorInterval"; cfg.ptr = &tsMonitorInterval; @@ -600,7 +590,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 600; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "offlineThreshold"; cfg.ptr = &tsOfflineThreshold; @@ -610,7 +600,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 86400 * 365; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcTimer"; cfg.ptr = &tsRpcTimer; @@ -620,7 +610,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 3000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcForceTcp"; cfg.ptr = &tsRpcForceTcp; @@ -630,7 +620,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcMaxTime"; cfg.ptr = &tsRpcMaxTime; @@ -640,7 +630,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 7200; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; @@ -650,7 +640,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "shellActivityTimer"; cfg.ptr = &tsShellActivityTimer; @@ -660,7 +650,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 120; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; @@ -670,7 +660,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minIntervalTime"; cfg.ptr = &tsMinIntervalTime; @@ -680,7 +670,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxStreamCompDelay"; cfg.ptr = &tsMaxStreamComputDelay; @@ -690,7 +680,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxFirstStreamCompDelay"; cfg.ptr = &tsStreamCompStartDelay; @@ -700,7 +690,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "retryStreamCompDelay"; cfg.ptr = &tsRetryStreamCompDelay; @@ -711,7 +701,7 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "streamCompDelayRatio"; cfg.ptr = &tsStreamComputDelayRatio; cfg.valType = TAOS_CFG_VTYPE_FLOAT; @@ -720,7 +710,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0.9f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxVgroupsPerDb"; cfg.ptr = &tsMaxVgroupsPerDb; @@ -730,7 +720,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 8192; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cache"; cfg.ptr = &tsCacheBlockSize; @@ -740,7 +730,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "blocks"; cfg.ptr = &tsBlocksPerVnode; @@ -750,7 +740,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "days"; cfg.ptr = &tsDaysPerFile; @@ -760,7 +750,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "keep"; cfg.ptr = &tsDaysToKeep; @@ -770,7 +760,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_KEEP; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minRows"; cfg.ptr = &tsMinRowsInFileBlock; @@ -780,7 +770,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRows"; cfg.ptr = &tsMaxRowsInFileBlock; @@ -790,7 +780,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "precision"; cfg.ptr = &tsTimePrecision; @@ -800,7 +790,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_PRECISION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "comp"; cfg.ptr = &tsCompression; @@ -810,7 +800,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_COMP_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "walLevel"; cfg.ptr = &tsWAL; @@ -820,7 +810,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_WAL_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fsync"; cfg.ptr = &tsFsyncPeriod; @@ -830,7 +820,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FSYNC_PERIOD; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "replica"; cfg.ptr = &tsReplications; @@ -840,7 +830,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "quorum"; cfg.ptr = &tsQuorum; @@ -850,7 +840,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "update"; cfg.ptr = &tsUpdate; @@ -860,7 +850,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_UPDATE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cachelast"; cfg.ptr = &tsCacheLastRow; @@ -870,7 +860,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_CACHE_LAST_ROW; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttHostName"; cfg.ptr = tsMqttHostName; @@ -880,7 +870,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_HOSTNAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttPort"; cfg.ptr = tsMqttPort; @@ -890,7 +880,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_PORT_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttTopic"; cfg.ptr = tsMqttTopic; @@ -900,7 +890,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_TOPIC_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "compressMsgSize"; cfg.ptr = &tsCompressMsgSize; @@ -910,7 +900,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "compressColData"; cfg.ptr = &tsCompressColData; @@ -920,7 +910,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxSQLLength"; cfg.ptr = &tsMaxSQLStringLen; @@ -930,7 +920,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxWildCardsLength"; cfg.ptr = &tsMaxWildCardsLen; @@ -940,7 +930,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRegexStringLen"; cfg.ptr = &tsMaxRegexStringLen; @@ -950,7 +940,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxNumOfOrderedRes"; cfg.ptr = &tsMaxNumOfOrderedResults; @@ -960,7 +950,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "queryBufferSize"; cfg.ptr = &tsQueryBufferSize; @@ -970,7 +960,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 500000000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "retrieveBlockingModel"; cfg.ptr = &tsRetrieveBlockingModel; @@ -980,7 +970,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "keepColumnName"; cfg.ptr = &tsKeepOriginalColumnName; @@ -990,7 +980,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // locale & charset cfg.option = "timezone"; @@ -1001,7 +991,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_TIMEZONE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "locale"; cfg.ptr = tsLocale; @@ -1011,7 +1001,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "charset"; cfg.ptr = tsCharset; @@ -1021,7 +1011,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // connect configs cfg.option = "maxShellConns"; @@ -1032,7 +1022,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 50000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxConnections"; cfg.ptr = &tsMaxConnections; @@ -1042,7 +1032,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalLogDirGB"; cfg.ptr = &tsMinimalLogDirGB; @@ -1052,7 +1042,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalTmpDirGB"; cfg.ptr = &tsReservedTmpDirectorySpace; @@ -1062,7 +1052,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalDataDirGB"; cfg.ptr = &tsMinimalDataDirGB; @@ -1072,18 +1062,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); - - // module configs - cfg.option = "mnodeEqualVnodeNum"; - cfg.ptr = &tsMnodeEqualVnodeNum; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // module configs cfg.option = "flowctrl"; @@ -1094,7 +1073,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "slaveQuery"; cfg.ptr = &tsEnableSlaveQuery; @@ -1104,7 +1083,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "adjustMaster"; cfg.ptr = &tsEnableAdjustMaster; @@ -1114,17 +1093,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "http"; - cfg.ptr = &tsEnableHttpModule; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqtt"; cfg.ptr = &tsEnableMqttModule; @@ -1134,7 +1103,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monitor"; cfg.ptr = &tsEnableMonitorModule; @@ -1144,7 +1113,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "stream"; cfg.ptr = &tsEnableStream; @@ -1154,7 +1123,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "topicBianryLen"; cfg.ptr = &tsTopicBianryLen; @@ -1164,17 +1133,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 16000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpEnableRecordSql"; - cfg.ptr = &tsHttpEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "telegrafUseFieldNum"; cfg.ptr = &tsTelegrafUseFieldNum; @@ -1184,17 +1143,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpMaxThreads"; - cfg.ptr = &tsHttpMaxThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 2; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "restfulRowLimit"; cfg.ptr = &tsRestRowLimit; @@ -1204,17 +1153,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpDbNameMandatory"; - cfg.ptr = &tsHttpDbNameMandatory; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // debug flag cfg.option = "numOfLogLines"; @@ -1225,7 +1164,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "logKeepDays"; cfg.ptr = &tsLogKeepDays; @@ -1235,7 +1174,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 365000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; @@ -1245,7 +1184,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "debugFlag"; cfg.ptr = &debugFlag; @@ -1255,7 +1194,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mDebugFlag"; cfg.ptr = &mDebugFlag; @@ -1265,7 +1204,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dDebugFlag"; cfg.ptr = &dDebugFlag; @@ -1275,7 +1214,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "sDebugFlag"; cfg.ptr = &sDebugFlag; @@ -1285,7 +1224,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "wDebugFlag"; cfg.ptr = &wDebugFlag; @@ -1295,7 +1234,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "sdbDebugFlag"; @@ -1306,7 +1245,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcDebugFlag"; cfg.ptr = &rpcDebugFlag; @@ -1316,7 +1255,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tmrDebugFlag"; cfg.ptr = &tmrDebugFlag; @@ -1326,7 +1265,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cDebugFlag"; cfg.ptr = &cDebugFlag; @@ -1336,7 +1275,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "jniDebugFlag"; cfg.ptr = &jniDebugFlag; @@ -1346,7 +1285,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "odbcDebugFlag"; cfg.ptr = &odbcDebugFlag; @@ -1356,7 +1295,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "uDebugFlag"; cfg.ptr = &uDebugFlag; @@ -1366,7 +1305,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "httpDebugFlag"; cfg.ptr = &httpDebugFlag; @@ -1376,7 +1315,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttDebugFlag"; cfg.ptr = &mqttDebugFlag; @@ -1386,7 +1325,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monDebugFlag"; cfg.ptr = &monDebugFlag; @@ -1396,7 +1335,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "qDebugFlag"; cfg.ptr = &qDebugFlag; @@ -1406,7 +1345,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "vDebugFlag"; cfg.ptr = &vDebugFlag; @@ -1416,7 +1355,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tsdbDebugFlag"; cfg.ptr = &tsdbDebugFlag; @@ -1426,7 +1365,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cqDebugFlag"; cfg.ptr = &cqDebugFlag; @@ -1436,7 +1375,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; @@ -1446,7 +1385,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "enableCoreFile"; cfg.ptr = &tsEnableCoreFile; @@ -1456,7 +1395,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // version info cfg.option = "gitinfo"; @@ -1467,7 +1406,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "gitinfoOfInternal"; cfg.ptr = gitinfoOfInternal; @@ -1477,7 +1416,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "buildinfo"; cfg.ptr = buildinfo; @@ -1487,7 +1426,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "version"; cfg.ptr = version; @@ -1497,7 +1436,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxBinaryDisplayWidth"; cfg.ptr = &tsMaxBinaryDisplayWidth; @@ -1507,7 +1446,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tempDir"; cfg.ptr = tsTempDir; @@ -1517,7 +1456,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = PATH_MAX; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tsdbMetaCompactRatio"; cfg.ptr = &tsTsdbMetaCompactRatio; @@ -1527,7 +1466,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // enable kill long query cfg.option = "deadLockKillQuery"; @@ -1538,7 +1477,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); #ifdef TD_TSZ // lossy compress @@ -1550,7 +1489,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = tListLen(lossyColumns); cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fPrecision"; cfg.ptr = &fPrecision; @@ -1562,7 +1501,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dPrecision"; cfg.ptr = &dPrecision; @@ -1572,7 +1511,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = MAX_FLOAT; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRange"; cfg.ptr = &maxRange; @@ -1582,7 +1521,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "range"; cfg.ptr = &curRange; @@ -1592,10 +1531,10 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); #else - assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); + //assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); #endif } @@ -1604,7 +1543,7 @@ void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -int32_t taosCheckGlobalCfg() { +int32_t taosCheckAndPrintCfg() { char fqdn[TSDB_FQDN_LEN]; uint16_t port; @@ -1652,37 +1591,13 @@ int32_t taosCheckGlobalCfg() { tsNumOfCores = 1; } - if (tsHttpMaxThreads == 2) { - int32_t halfNumOfCores = tsNumOfCores >> 1; - if (halfNumOfCores > 2) { - tsHttpMaxThreads = halfNumOfCores; - } - } - - // todo refactor - tsVersion = 0; - for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) { - if (version[i] >= '0' && version[i] <= '9') { - ver = ver * 10 + (version[i] - '0'); - } else if (version[i] == '.') { - tsVersion |= ver & 0xFF; - tsVersion <<= 8; - - ver = 0; - } else if (version[i] == 0) { - tsVersion |= ver & 0xFF; - - break; - } - } - if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } uInfo(" check global cfg completed"); uInfo("=================================="); - taosPrintGlobalCfg(); + taosPrintCfg(); return 0; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 6c795bb0cc1d8e34616553ac0a79c5a0175558b6..5c6b9ff6f33e382f10fb2045df9e5ec432b28732 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -82,18 +82,18 @@ void deltaToUtcInitOnce() { } static int64_t parseFraction(char* str, char** end, int32_t timePrec); -static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec, char delim); +static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec); static int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec); static char* forwardToTimeStringEnd(char* str); -static bool checkTzPresent(char *str, int32_t len); +static bool checkTzPresent(const char *str, int32_t len); static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = { parseLocaltime, parseLocaltimeDst }; -int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { +int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { return parseTimeWithTz(timestr, time, timePrec, 'T'); @@ -104,7 +104,7 @@ int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePre } } -bool checkTzPresent(char *str, int32_t len) { +bool checkTzPresent(const char *str, int32_t len) { char *seg = forwardToTimeStringEnd(str); int32_t seg_len = len - (int32_t)(seg - str); @@ -237,7 +237,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { * 2013-04-12T15:52:01+0800 * 2013-04-12T15:52:01.123+0800 */ -int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec, char delim) { +int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); @@ -432,7 +432,7 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time * d - Days (24 hours) * w - Weeks (7 days) */ -int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { +int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { errno = 0; char* endPtr = NULL; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index ed1f4d2bfd7826514a5d9a18665f33d8b01aed54..8a5a300fcfb79dd6f04f1d203469ebe827bb5cc8 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -75,7 +75,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* return 0; } -void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) { +void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { int32_t ret = 0; memset(pVar, 0, sizeof(SVariant)); diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index cf9a9600622aa2b5f479f239e3a24553a44ab1be..0a725d9975c4a9379825169a7d895366fb2d7a5f 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -112,12 +112,12 @@ int dmnReadConfig(const char *path) { return -1; } - if (taosReadGlobalCfg() != 0) { + if (taosReadCfgFromFile() != 0) { uError("failed to read global config"); return -1; } - if (taosCheckGlobalCfg() != 0) { + if (taosCheckAndPrintCfg() != 0) { uError("failed to check global config"); return -1; } @@ -142,7 +142,6 @@ void dmnInitOption(SDnodeOpt *pOption) { pOption->numOfSupportVnodes = 1; pOption->numOfSupportQnodes = 1; pOption->statusInterval = tsStatusInterval; - pOption->mnodeEqualVnodeNum = tsMnodeEqualVnodeNum; pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; pOption->ratioOfQueryCores = tsRatioOfQueryCores; pOption->maxShellConns = tsMaxShellConns; diff --git a/source/dnode/mgmt/impl/inc/dndDnode.h b/source/dnode/mgmt/impl/inc/dndDnode.h index c21c6a0b860514ce815f4628d1cfc946d68b3123..27cc99c27cb0f5206674215a107838d8a3b3a7ce 100644 --- a/source/dnode/mgmt/impl/inc/dndDnode.h +++ b/source/dnode/mgmt/impl/inc/dndDnode.h @@ -31,6 +31,7 @@ int32_t dndGetClusterId(SDnode *pDnode); void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort); void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet); void dndSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg); +void dndSendStatusMsg(SDnode *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 136f6eee0c823c461b768ccf75e7f7e7f78b832c..d8acda530ddd1733f0ba82aa0d37c578bca41bda 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -58,7 +58,8 @@ typedef struct { int32_t dnodeId; int32_t dropped; int32_t clusterId; - uint32_t rebootTime; + int64_t rebootTime; + int8_t statusSent; SEpSet mnodeEpSet; char *file; SHashObj *dnodeHash; @@ -111,6 +112,7 @@ typedef struct SDnode { EStat stat; SDnodeOpt opt; SDnodeDir dir; + FileFd lockFd; SDnodeMgmt dmgmt; SMnodeMgmt mmgmt; SVnodesMgmt vmgmt; diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index c89741d03ba44e47bdeb9b93e1a85935514bd597..9e2d4d4c9b00417ce0ccf91042c93d8e41a1f962 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -17,6 +17,7 @@ #include "dndDnode.h" #include "dndTransport.h" #include "dndVnodes.h" +#include "tep.h" int32_t dndGetDnodeId(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; @@ -281,9 +282,8 @@ PRASE_DNODE_OVER: if (pMgmt->dnodeEps == NULL) { pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp)); pMgmt->dnodeEps->num = 1; - pMgmt->dnodeEps->eps[0].isMnode = 1; - pMgmt->dnodeEps->eps[0].port = pDnode->opt.serverPort; - tstrncpy(pMgmt->dnodeEps->eps[0].fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN); + pMgmt->dnodeEps->eps[0].isMnode = 1; + taosGetFqdnPortFromEp(pDnode->opt.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port); } dndResetDnodes(pDnode, pMgmt->dnodeEps); @@ -335,7 +335,7 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { return 0; } -static void dndSendStatusMsg(SDnode *pDnode) { +void dndSendStatusMsg(SDnode *pDnode) { int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); SStatusMsg *pStatus = rpcMallocCont(contLen); @@ -349,7 +349,7 @@ static void dndSendStatusMsg(SDnode *pDnode) { pStatus->sver = htonl(pDnode->opt.sver); pStatus->dnodeId = htonl(pMgmt->dnodeId); pStatus->clusterId = htonl(pMgmt->clusterId); - pStatus->rebootTime = htonl(pMgmt->rebootTime); + pStatus->rebootTime = htobe64(pMgmt->rebootTime); pStatus->numOfCores = htons(pDnode->opt.numOfCores); pStatus->numOfSupportMnodes = htons(pDnode->opt.numOfCores); pStatus->numOfSupportVnodes = htons(pDnode->opt.numOfCores); @@ -357,7 +357,6 @@ static void dndSendStatusMsg(SDnode *pDnode) { tstrncpy(pStatus->dnodeEp, pDnode->opt.localEp, TSDB_EP_LEN); pStatus->clusterCfg.statusInterval = htonl(pDnode->opt.statusInterval); - pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(pDnode->opt.mnodeEqualVnodeNum); pStatus->clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); @@ -371,6 +370,9 @@ static void dndSendStatusMsg(SDnode *pDnode) { contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad); SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS}; + pMgmt->statusSent = 1; + + dTrace("pDnode:%p, send status msg to mnode", pDnode); dndSendMsgToMnode(pDnode, &rpcMsg); } @@ -383,7 +385,7 @@ static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { pMgmt->dnodeId = pCfg->dnodeId; pMgmt->clusterId = pCfg->clusterId; pMgmt->dropped = pCfg->dropped; - (void)dndWriteDnodes(pDnode); + dndWriteDnodes(pDnode); taosWUnLockLatch(&pMgmt->latch); } } @@ -409,11 +411,16 @@ static void dndUpdateDnodeEps(SDnode *pDnode, SDnodeEps *pDnodeEps) { } static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SDnodeMgmt *pMgmt = &pDnode->dmgmt; + if (pEpSet && pEpSet->numOfEps > 0) { dndUpdateMnodeEpSet(pDnode, pEpSet); } - if (pMsg->code != TSDB_CODE_SUCCESS) return; + if (pMsg->code != TSDB_CODE_SUCCESS) { + pMgmt->statusSent = 0; + return; + } SStatusRsp *pRsp = pMsg->pCont; SDnodeCfg *pCfg = &pRsp->dnodeCfg; @@ -421,7 +428,10 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { pCfg->clusterId = htonl(pCfg->clusterId); dndUpdateDnodeCfg(pDnode, pCfg); - if (pCfg->dropped) return; + if (pCfg->dropped) { + pMgmt->statusSent = 0; + return; + } SDnodeEps *pDnodeEps = &pRsp->dnodeEps; pDnodeEps->num = htonl(pDnodeEps->num); @@ -431,6 +441,7 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { } dndUpdateDnodeEps(pDnode, pDnodeEps); + pMgmt->statusSent = 0; } static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); } @@ -438,13 +449,12 @@ static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { a static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); } static void dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg) { - dDebug("config msg is received"); + dError("config msg is received, but not supported yet"); SCfgDnodeMsg *pCfg = pMsg->pCont; int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code}; rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); } static void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) { @@ -457,18 +467,18 @@ static void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)}; rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); } static void *dnodeThreadRoutine(void *param) { - SDnode *pDnode = param; - int32_t ms = pDnode->opt.statusInterval * 1000; + SDnode *pDnode = param; + SDnodeMgmt *pMgmt = &pDnode->dmgmt; + int32_t ms = pDnode->opt.statusInterval * 1000; while (true) { - taosMsleep(ms); pthread_testcancel(); + taosMsleep(ms); - if (dndGetStat(pDnode) == DND_STAT_RUNNING) { + if (dndGetStat(pDnode) == DND_STAT_RUNNING && !pMgmt->statusSent) { dndSendStatusMsg(pDnode); } } @@ -478,7 +488,7 @@ int32_t dndInitDnode(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; pMgmt->dnodeId = 0; - pMgmt->rebootTime = taosGetTimestampSec(); + pMgmt->rebootTime = taosGetTimestampMs(); pMgmt->dropped = 0; pMgmt->clusterId = 0; @@ -556,8 +566,10 @@ void dndProcessDnodeReq(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { dError("RPC %p, dnode req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED}; rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); } + + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } void dndProcessDnodeRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -574,4 +586,7 @@ void dndProcessDnodeRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { default: dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); } + + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index d3336fb079a5007a8a7ad628df21ea4283b29619..3cf08e619ea4fd80a26f739f2b1248c490c0dd43 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -334,7 +334,6 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->cfg.sver = pDnode->opt.sver; pOption->cfg.enableTelem = pDnode->opt.enableTelem; pOption->cfg.statusInterval = pDnode->opt.statusInterval; - pOption->cfg.mnodeEqualVnodeNum = pDnode->opt.mnodeEqualVnodeNum; pOption->cfg.shellActivityTimer = pDnode->opt.shellActivityTimer; pOption->cfg.timezone = pDnode->opt.timezone; pOption->cfg.charset = pDnode->opt.charset; @@ -560,9 +559,12 @@ static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { break; } - SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; taosFreeQitem(pMsg); } @@ -646,9 +648,12 @@ void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); if (pMsg == NULL || taosWriteQitem(pMgmt->pMgmtQ, pMsg) != 0) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; - rpcSendResponse(&rsp); + if (pRpcMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; + rpcSendResponse(&rsp); + } rpcFreeCont(pRpcMsg->pCont); + pRpcMsg->pCont = NULL; taosFreeQitem(pMsg); } } @@ -657,9 +662,12 @@ void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } dndReleaseMnode(pDnode, pMnode); @@ -669,9 +677,12 @@ void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } dndReleaseMnode(pDnode, pMnode); @@ -681,9 +692,12 @@ void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } dndReleaseMnode(pDnode, pMnode); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 0f8281a3a61e53c900f75e81d256a376343224c3..6dc46cefcdd4260df881d57126ae7773d98964b9 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -69,9 +69,9 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_RETRIEVE_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_VGROUP_LIST] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; @@ -84,12 +84,12 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq; // message from mnode to vnode - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB_IN_RSP] = dndProcessMnodeWriteMsg; // message from mnode to dnode pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dndProcessVnodeMgmtMsg; @@ -130,7 +130,7 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (dndGetStat(pDnode) == DND_STAT_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; - dTrace("RPC %p, rsp:%s app:%p is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]); rpcFreeCont(pMsg->pCont); return; } @@ -138,12 +138,11 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { DndMsgFp fp = pMgmt->msgFp[msgType]; if (fp != NULL) { (*fp)(pDnode, pMsg, pEpSet); - dTrace("RPC %p, rsp:%s app:%p is processed, code:0x%0X", pMsg->handle, taosMsg[msgType], pMsg->ahandle, - pMsg->code & 0XFFFF); + dTrace("RPC %p, rsp:%s is processed, code:0x%0X", pMsg->handle, taosMsg[msgType], pMsg->code & 0XFFFF); } else { - dError("RPC %p, rsp:%s app:%p not processed", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]); + rpcFreeCont(pMsg->pCont); } - rpcFreeCont(pMsg->pCont); } static int32_t dndInitClient(SDnode *pDnode) { @@ -236,18 +235,18 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp static int32_t dndAuthInternalMsg(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, INTERNAL_USER) == 0) { // A simple temporary implementation - char pass[32] = {0}; + char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - memcpy(secret, pass, TSDB_KEY_LEN); + memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 0; *encrypt = 0; *ckey = 0; return 0; } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { // A simple temporary implementation - char pass[32] = {0}; + char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - memcpy(secret, pass, TSDB_KEY_LEN); + memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 0; *encrypt = 0; *ckey = 0; @@ -289,8 +288,8 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); } else { SAuthRsp *pRsp = rpcRsp.pCont; - memcpy(secret, pRsp->secret, TSDB_KEY_LEN); - memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); + memcpy(secret, pRsp->secret, TSDB_PASSWORD_LEN); + memcpy(ckey, pRsp->ckey, TSDB_PASSWORD_LEN); *spi = pRsp->spi; *encrypt = pRsp->encrypt; dDebug("user:%s, success to get user auth from other mnodes", user); @@ -369,4 +368,4 @@ void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pMsg) { SEpSet epSet = {0}; dndGetMnodeEpSet(pDnode, &epSet); dndSendMsgToDnode(pDnode, &epSet, pMsg); -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index 1159bbb4c5fda40dcd45afdd740ad025ab250b8d..0dbee0d3379daa55e69ba6ed5e0b2beaa400dec7 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -55,30 +55,31 @@ void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup) { pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } -static int32_t dndCheckRunning(char *dataDir) { +static FileFd dndCheckRunning(char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); FileFd fd = taosOpenFileCreateWriteTrunc(filepath); if (fd < 0) { - dError("failed to lock file:%s since %s, quit", filepath, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s, quit", filepath, terrstr()); return -1; } int32_t ret = taosLockFile(fd); if (ret != 0) { - dError("failed to lock file:%s since %s, quit", filepath, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s, quit", filepath, terrstr()); taosCloseFile(fd); return -1; } - return 0; + return fd; } static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { - if (dndCheckRunning(pOption->dataDir) != 0) { + pDnode->lockFd = dndCheckRunning(pOption->dataDir); + if (pDnode->lockFd < 0) { return -1; } @@ -133,6 +134,12 @@ static void dndCleanupEnv(SDnode *pDnode) { tfree(pDnode->dir.dnode); } + if (pDnode->lockFd >= 0) { + taosUnLockFile(pDnode->lockFd); + taosCloseFile(pDnode->lockFd); + pDnode->lockFd = 0; + } + taosStopCacheRefreshWorker(); } @@ -194,13 +201,16 @@ SDnode *dndInit(SDnodeOpt *pOption) { } dndSetStat(pDnode, DND_STAT_RUNNING); + dndSendStatusMsg(pDnode); dndReportStartup(pDnode, "TDengine", "initialized successfully"); - dInfo("TDengine is initialized successfully"); + dInfo("TDengine is initialized successfully, pDnode:%p", pDnode); return pDnode; } void dndCleanup(SDnode *pDnode) { + if (pDnode == NULL) return; + if (dndGetStat(pDnode) == DND_STAT_STOPPED) { dError("dnode is shutting down"); return; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index b3b6818a5c4709b72360fcfe68b2593643228a21..b340029044c7b0f23372010872aa06b2b75146a1 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,5 +1,6 @@ # add_subdirectory(acct) # add_subdirectory(cluster) +add_subdirectory(dnode) # add_subdirectory(profile) # add_subdirectory(show) add_subdirectory(user) diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index cda44a3be8f0c0313baaa26247bbf6bf3e5289ef..9050a938fa54634d20e7a8981b864c546ca53d9b 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -33,7 +33,7 @@ class DndTestAcct : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index 4e71a8016cfe8ec50dfc56bd62e179d26298fcd8..d47e63a85f40033aa623bc43d6104a458e91a8f4 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -33,7 +33,7 @@ class DndTestCluster : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } @@ -80,7 +80,7 @@ TEST_F(DndTestCluster, ShowCluster) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, "show cluster"); + EXPECT_STREQ(pMeta->tbFname, "show cluster"); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..e118cb872559eab22e1210d37fee11f30bf281c4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dndTestDnode "") + +target_sources(dndTestDnode + PRIVATE + "dnode.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dndTestDnode + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dndTestDnode + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dndTestDnode + COMMAND dndTestDnode +) diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fba3794f6abddebd80f0535e9d6b90082089e949 --- /dev/null +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestDnode : public ::testing::Test { + public: + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } + + static void SetUpTestSuite() { + initLog("/tmp/dndTestDnode"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9521"; + pServer1 = CreateServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); + pServer2 = CreateServer("/tmp/dndTestDnode2", fqdn, 9522, firstEp); + pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); + pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); + pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9521); + taosMsleep(300); + } + + static void TearDownTestSuite() { + stopServer(pServer1); + stopServer(pServer2); + stopServer(pServer3); + stopServer(pServer4); + stopServer(pServer5); + dropClient(pClient); + pServer1 = NULL; + pServer2 = NULL; + pServer3 = NULL; + pServer4 = NULL; + pServer5 = NULL; + pClient = NULL; + } + + static SServer* pServer1; + static SServer* pServer2; + static SServer* pServer3; + static SServer* pServer4; + static SServer* pServer5; + static SClient* pClient; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htons(pMeta->numOfTags); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); + } + + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_EQ(data, val); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(const char* val, int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; + +SServer* DndTestDnode::pServer1; +SServer* DndTestDnode::pServer2; +SServer* DndTestDnode::pServer3; +SServer* DndTestDnode::pServer4; +SServer* DndTestDnode::pServer5; +SClient* DndTestDnode::pClient; + +TEST_F(DndTestDnode, ShowDnode) { + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point"); + CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); + CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max vnodes"); + CheckSchema(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); + CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline reason"); + + SendThenCheckShowRetrieveMsg(1); + CheckInt16(1); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(1); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckBinary("", 24); +} + +TEST_F(DndTestDnode, ConfigDnode_01) { + SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg)); + pReq->dnodeId = htonl(1); + strcpy(pReq->config, "ddebugflag 131"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCfgDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CONFIG_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); +} + +TEST_F(DndTestDnode, CreateDnode_01) { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9522"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + SendThenCheckShowRetrieveMsg(2); + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9522", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(0); + CheckInt16(1); + CheckInt16(1); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("", 24); + CheckBinary("", 24); +} + +TEST_F(DndTestDnode, DropDnode_01) { + SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); + pReq->dnodeId = htonl(2); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + SendThenCheckShowRetrieveMsg(1); + CheckInt16(1); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(1); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckBinary("", 24); +} + +TEST_F(DndTestDnode, CreateDnode_02) { + { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9523"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9524"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9525"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + taosMsleep(1300); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + SendThenCheckShowRetrieveMsg(4); + CheckInt16(1); + CheckInt16(3); + CheckInt16(4); + CheckInt16(5); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9523", TSDB_EP_LEN); + CheckBinary("localhost:9524", TSDB_EP_LEN); + CheckBinary("localhost:9525", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(0); + CheckInt16(0); + CheckInt16(0); + CheckInt16(1); + CheckInt16(1); + CheckInt16(1); + CheckInt16(1); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("", 24); + CheckBinary("", 24); + CheckBinary("", 24); + CheckBinary("", 24); +} + +TEST_F(DndTestDnode, RestartDnode_01) { + uInfo("stop all server"); + stopServer(pServer1); + stopServer(pServer2); + stopServer(pServer3); + stopServer(pServer4); + stopServer(pServer5); + pServer1 = NULL; + pServer2 = NULL; + pServer3 = NULL; + pServer4 = NULL; + pServer5 = NULL; + + uInfo("start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9521"; + pServer1 = startServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); + // pServer1 = startServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); + // pServer1 = startServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); + // pServer1 = startServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + + uInfo("all server is running"); + + // taosMsleep(1300); + // SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + // SendThenCheckShowRetrieveMsg(4); + // CheckInt16(1); + // CheckInt16(3); + // CheckInt16(4); + // CheckInt16(5); + // CheckBinary("localhost:9521", TSDB_EP_LEN); + // CheckBinary("localhost:9523", TSDB_EP_LEN); + // CheckBinary("localhost:9524", TSDB_EP_LEN); + // CheckBinary("localhost:9525", TSDB_EP_LEN); + // CheckInt16(0); + // CheckInt16(0); + // CheckInt16(0); + // CheckInt16(0); + // CheckInt16(1); + // CheckInt16(1); + // CheckInt16(1); + // CheckInt16(1); + // CheckBinary("ready", 10); + // CheckBinary("ready", 10); + // CheckBinary("ready", 10); + // CheckBinary("ready", 10); + // CheckTimestamp(); + // CheckTimestamp(); + // CheckTimestamp(); + // CheckTimestamp(); + // CheckBinary("", 24); + // CheckBinary("", 24); + // CheckBinary("", 24); + // CheckBinary("", 24); +} diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index a4632547a2c34add2d69d972506c49dbba17d8d7..2cbb9d5c2c9b5cb8808f3941c47cb8c8e22bc542 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -33,7 +33,7 @@ class DndTestProfile : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } @@ -139,7 +139,7 @@ TEST_F(DndTestProfile, SConnectMsg_03) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_STREQ(pMeta->tbFname, ""); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); @@ -480,7 +480,7 @@ TEST_F(DndTestProfile, SKillQueryMsg_03) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_STREQ(pMeta->tbFname, ""); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); @@ -667,7 +667,7 @@ TEST_F(DndTestProfile, SKillStreamMsg_03) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_STREQ(pMeta->tbFname, ""); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index 266f5858e4c638cc2a3ff4c44aae9a17381bdefa..c535386ecda3eba092a8497a9aec68822b092a39 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -33,7 +33,7 @@ class DndTestShow : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } @@ -141,7 +141,7 @@ TEST_F(DndTestShow, SShowMsg_04) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_STREQ(pMeta->tbFname, ""); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 92b0967345eb2a94f5f059e041c70abe704cccd7..de50899c2d3c121f3ef34e764be5352cab91ca90 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -16,9 +16,9 @@ #include "deploy.h" void initLog(const char* path) { - dDebugFlag = 0; + dDebugFlag = 143; vDebugFlag = 0; - mDebugFlag = 207; + mDebugFlag = 143; cDebugFlag = 0; jniDebugFlag = 0; tmrDebugFlag = 0; @@ -26,7 +26,7 @@ void initLog(const char* path) { httpDebugFlag = 0; mqttDebugFlag = 0; monDebugFlag = 0; - uDebugFlag = 0; + uDebugFlag = 143; rpcDebugFlag = 0; odbcDebugFlag = 0; qDebugFlag = 0; @@ -34,6 +34,10 @@ void initLog(const char* path) { sDebugFlag = 0; tsdbDebugFlag = 0; cqDebugFlag = 0; + tscEmbeddedInUtil = 1; + + taosRemoveDir(path); + taosMkDir(path); char temp[PATH_MAX]; snprintf(temp, PATH_MAX, "%s/taosdlog", path); @@ -50,14 +54,13 @@ void* runServer(void* param) { } } -void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port) { +void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port, const char* firstEp) { pOption->sver = 1; pOption->numOfCores = 1; pOption->numOfSupportMnodes = 1; pOption->numOfSupportVnodes = 1; pOption->numOfSupportQnodes = 1; pOption->statusInterval = 1; - pOption->mnodeEqualVnodeNum = 1; pOption->numOfThreadsPerCore = 1; pOption->ratioOfQueryCores = 1; pOption->maxShellConns = 1000; @@ -66,16 +69,14 @@ void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t strcpy(pOption->dataDir, path); snprintf(pOption->localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); snprintf(pOption->localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(pOption->firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(pOption->firstEp, TSDB_EP_LEN, "%s", firstEp); } -SServer* createServer(const char* path, const char* fqdn, uint16_t port) { - taosRemoveDir(path); +SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { taosMkDir(path); - initLog(path); SDnodeOpt option = {0}; - initOption(&option, path, fqdn, port); + initOption(&option, path, fqdn, port, firstEp); SDnode* pDnode = dndInit(&option); ASSERT(pDnode); @@ -90,16 +91,28 @@ SServer* createServer(const char* path, const char* fqdn, uint16_t port) { return pServer; } -void dropServer(SServer* pServer) { +SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + taosRemoveDir(path); + return startServer(path, fqdn, port, firstEp); +} + +void stopServer(SServer* pServer) { + if (pServer == NULL) return; if (pServer->threadId != NULL) { taosDestoryThread(pServer->threadId); } + + if (pServer->pDnode != NULL) { + dndCleanup(pServer->pDnode); + pServer->pDnode = NULL; + } } void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SClient* pClient = (SClient*)parent; pClient->pRsp = pMsg; - // taosMsleep(1000000); + uInfo("response:%s from dnode, pCont:%p contLen:%d code:0x%X", taosMsg[pMsg->msgType], pMsg->pCont, pMsg->contLen, + pMsg->code); tsem_post(&pClient->sem); } @@ -107,7 +120,7 @@ SClient* createClient(const char* user, const char* pass, const char* fqdn, uint SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); ASSERT(pClient); - char secretEncrypt[32] = {0}; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit; @@ -145,7 +158,7 @@ void sendMsg(SClient* pClient, SRpcMsg* pMsg) { epSet.inUse = 0; epSet.numOfEps = 1; epSet.port[0] = pClient->port; - strcpy(epSet.fqdn[0], pClient->fqdn); + memcpy(epSet.fqdn[0], pClient->fqdn, TSDB_FQDN_LEN); rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL); tsem_wait(&pClient->sem); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index 93188458eca37e159fa9be932797ecc5baa13644..4c082df5f3b1592b474e1d62fa0068f28bde1233 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -39,8 +39,10 @@ typedef struct { tsem_t sem; } SClient; -SServer* createServer(const char* path, const char* fqdn, uint16_t port); -void dropServer(SServer* pServer); +void initLog(const char* path); +SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); +SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); +void stopServer(SServer* pServer); SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); void dropClient(SClient* pClient); void sendMsg(SClient* pClient, SRpcMsg* pMsg); diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 36f801fec20eefc4cecea44421de29267580752b..48be2635cddb2903d58e8d10ee1263bb0355e48c 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -17,269 +17,223 @@ class DndTestUser : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestUser"; - const char* fqdn = "localhost"; - uint16_t port = 9524; + initLog("/tmp/dndTestUser"); - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9530"; + pServer = CreateServer("/tmp/dndTestUser", fqdn, 9530, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9530); + taosMsleep(300); } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); + pServer = NULL; + pClient = NULL; } static SServer* pServer; static SClient* pClient; static int32_t connId; -}; -SServer* DndTestUser::pServer; -SClient* DndTestUser::pClient; -int32_t DndTestUser::connId; + public: + void SetUp() override {} + void TearDown() override {} -#if 0 -TEST_F(DndTestUser, ShowUser) { - int32_t showId = 0; - - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - strcpy(pShow->db, ""); - - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, "show users"); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 4); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "name"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, 10 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "privilege"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "create_time"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "account"); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, 2); - EXPECT_EQ(pRetrieveRsp->offset, 0); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - EXPECT_EQ(pRetrieveRsp->completed, 1); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - int64_t int64Val = 0; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htons(pMeta->numOfTags); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); } - //--- privilege --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += 10; - EXPECT_STREQ(strVal, "super"); + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += 10; - EXPECT_STREQ(strVal, "writable"); + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); } - //--- create_time --- - { - int64Val = *((int64_t*)(pData + pos)); + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); pos += sizeof(int64_t); - EXPECT_GT(int64Val, 0); + EXPECT_EQ(data, val); + } - int64Val = *((int64_t*)(pData + pos)); + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); pos += sizeof(int64_t); - EXPECT_GT(int64Val, 0); + EXPECT_GT(data, 0); } - //--- account --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - + void CheckBinary(const char* val, int32_t len) { pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); } -} -#endif - -TEST_F(DndTestUser, CreateUser_01) { - ASSERT_NE(pClient, nullptr); - //--- create user --- - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p1"); + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; +SServer* DndTestUser::pServer; +SClient* DndTestUser::pClient; +int32_t DndTestUser::connId; - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); +TEST_F(DndTestUser, ShowUser) { + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); + CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); + + SendThenCheckShowRetrieveMsg(1); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); +} - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 3); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- +TEST_F(DndTestUser, CreateUser_01) { { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "u1"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); + { + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u2"); + strcpy(pReq->pass, "p2"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(3); + CheckBinary("u1", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("normal", 10); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); } TEST_F(DndTestUser, AlterUser_01) { - ASSERT_NE(pClient, nullptr); - - //--- drop user --- SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p2"); @@ -294,60 +248,23 @@ TEST_F(DndTestUser, AlterUser_01) { ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 3); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "u1"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(3); + CheckBinary("u1", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("normal", 10); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); } TEST_F(DndTestUser, DropUser_01) { - ASSERT_NE(pClient, nullptr); - - //--- drop user --- SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); strcpy(pReq->user, "u1"); @@ -361,47 +278,38 @@ TEST_F(DndTestUser, DropUser_01) { ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 2); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(2); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); +} - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } -} \ No newline at end of file +TEST_F(DndTestUser, RestartDnode) { + stopServer(pServer); + pServer = NULL; + + uInfo("start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9530"; + pServer = startServer("/tmp/dndTestUser", fqdn, 9530, firstEp); + + uInfo("all server is running"); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(2); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); +} diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7a7bc161da36c224b7ce32488139e393607fb1e7..5ddde3181eb64fa0ddee76a9daeac13b6fd5abda 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -92,7 +92,6 @@ typedef enum { DND_REASON_VERSION_NOT_MATCH, DND_REASON_DNODE_ID_NOT_MATCH, DND_REASON_CLUSTER_ID_NOT_MATCH, - DND_REASON_MN_EQUAL_VN_NOT_MATCH, DND_REASON_STATUS_INTERVAL_NOT_MATCH, DND_REASON_TIME_ZONE_NOT_MATCH, DND_REASON_LOCALE_NOT_MATCH, @@ -125,6 +124,7 @@ typedef struct SDnodeObj { int64_t createdTime; int64_t updateTime; int64_t rebootTime; + int64_t lastAccessTime; int32_t accessTimes; int16_t numOfMnodes; int16_t numOfVnodes; @@ -180,13 +180,11 @@ typedef struct SAcctObj { typedef struct SUserObj { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; - int8_t superAuth; - int8_t readAuth; - int8_t writeAuth; + int8_t superUser; int32_t acctId; SHashObj *prohibitDbHash; } SUserObj; @@ -241,7 +239,7 @@ typedef struct SVgObj { SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; -typedef struct SStableObj { +typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_FULL_DB_NAME_LEN]; int64_t createdTime; @@ -251,9 +249,8 @@ typedef struct SStableObj { int32_t numOfColumns; int32_t numOfTags; SRWLatch lock; - SSchema *columnSchema; - SSchema *tagSchema; -} SStableObj; + SSchema *pSchema; +} SStbObj; typedef struct SFuncObj { char name[TSDB_FUNC_NAME_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index b44d6570fecd0999595a570f78a08380539e4e1a..e9913803bd7f2fe88db87c2ea2b44aafb062f554 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -88,6 +88,8 @@ void mndSendMsgToMnode(SMnode *pMnode, SRpcMsg *pMsg); void mndSendRedirectMsg(SMnode *pMnode, SRpcMsg *pMsg); void mndSetMsgHandle(SMnode *pMnode, int32_t msgType, MndMsgFp fp); +uint64_t mndGenerateUid(char *name, int32_t len) ; + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndStable.h b/source/dnode/mnode/impl/inc/mndStb.h similarity index 81% rename from source/dnode/mnode/impl/inc/mndStable.h rename to source/dnode/mnode/impl/inc/mndStb.h index c7767a59e445faabdb7f873f87021b7e660e59ff..58cae73c7fa0e58fbd7bd82d55adfaa06fc345cc 100644 --- a/source/dnode/mnode/impl/inc/mndStable.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MND_STABLE_H_ -#define _TD_MND_STABLE_H_ +#ifndef _TD_MND_STB_H_ +#define _TD_MND_STB_H_ #include "mndInt.h" @@ -22,11 +22,11 @@ extern "C" { #endif -int32_t mndInitStable(SMnode *pMnode); -void mndCleanupStable(SMnode *pMnode); +int32_t mndInitStb(SMnode *pMnode); +void mndCleanupStb(SMnode *pMnode); #ifdef __cplusplus } #endif -#endif /*_TD_MND_STABLE_H_*/ +#endif /*_TD_MND_STB_H_*/ diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index c5c71d8751d2bab9be1b9072d3fc70e79453ce4b..7edeb736470faead055430ee02f3b1d54acafddf 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -159,7 +159,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg cols++; pMeta->numOfColumns = htons(cols); - strcpy(pMeta->tableFname, "show cluster"); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 7477213890716651627bdc7b0f2aafdaf5f07398..fd02c6e25182f96d27430f6b9517cffddcad0c25 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -767,7 +767,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe pShow->numOfRows = sdbGetSize(pSdb, SDB_DB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index dc9f54888f6cf099373b7773ef46b27b4b2606a9..cf46d044cafdaa370736710c1f879add24023716 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -22,10 +22,13 @@ #include "tutil.h" #define TSDB_DNODE_VER 1 +#define TSDB_DNODE_RESERVE_SIZE 64 #define TSDB_CONFIG_OPTION_LEN 16 #define TSDB_CONIIG_VALUE_LEN 48 #define TSDB_CONFIG_NUMBER 8 +static int32_t id = 2; + static const char *offlineReason[] = { "", "status msg timeout", @@ -33,7 +36,6 @@ static const char *offlineReason[] = { "version not match", "dnodeId not match", "clusterId not match", - "mnEqualVn not match", "interval not match", "timezone not match", "locale not match", @@ -117,6 +119,7 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { SDB_SET_INT64(pRaw, dataPos, pDnode->updateTime) SDB_SET_INT16(pRaw, dataPos, pDnode->port) SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -142,28 +145,28 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pDnode->updateTime) SDB_GET_INT16(pRaw, pRow, dataPos, &pDnode->port) SDB_GET_BINARY(pRaw, pRow, dataPos, pDnode->fqdn, TSDB_FQDN_LEN) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_DNODE_RESERVE_SIZE) return pRow; } -static void mnodeResetDnode(SDnodeObj *pDnode) { +static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { + mTrace("dnode:%d, perform insert action", pDnode->id); + pDnode->rebootTime = 0; + pDnode->lastAccessTime = 0; pDnode->accessTimes = 0; - pDnode->numOfCores = 0; pDnode->numOfMnodes = 0; pDnode->numOfVnodes = 0; pDnode->numOfQnodes = 0; pDnode->numOfSupportMnodes = 0; pDnode->numOfSupportVnodes = 0; pDnode->numOfSupportQnodes = 0; + pDnode->numOfCores = 0; pDnode->status = DND_STATUS_OFFLINE; pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); -} -static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { - mTrace("dnode:%d, perform insert action", pDnode->id); - mnodeResetDnode(pDnode); return 0; } @@ -174,11 +177,6 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) { mTrace("dnode:%d, perform update action", pOldDnode->id); - pOldDnode->id = pNewDnode->id; - pOldDnode->createdTime = pNewDnode->createdTime; - pOldDnode->updateTime = pNewDnode->updateTime; - pOldDnode->port = pNewDnode->port; - memcpy(pOldDnode->fqdn, pNewDnode->fqdn, TSDB_FQDN_LEN); return 0; } @@ -232,6 +230,7 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { if (pIter == NULL) break; if (i >= numOfEps) { sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pDnode); break; } @@ -244,20 +243,15 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { pEp->isMnode = 1; } i++; + sdbRelease(pSdb, pDnode); } pEps->num = htonl(i); } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { - if (pCfg->mnodeEqualVnodeNum != pMnode->cfg.mnodeEqualVnodeNum) { - mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg inconsistent", pCfg->mnodeEqualVnodeNum, - pMnode->cfg.mnodeEqualVnodeNum); - return DND_REASON_MN_EQUAL_VN_NOT_MATCH; - } - if (pCfg->statusInterval != pMnode->cfg.statusInterval) { - mError("\"statusInterval\"[%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); + mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } @@ -265,18 +259,18 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) { - mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, + mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, pCfg->checkTime, checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } if (0 != strcasecmp(pCfg->locale, pMnode->cfg.locale)) { - mError("\"locale\"[%s - %s] cfg parameters inconsistent", pCfg->locale, pMnode->cfg.locale); + mError("locale [%s - %s] cfg inconsistent", pCfg->locale, pMnode->cfg.locale); return DND_REASON_LOCALE_NOT_MATCH; } if (0 != strcasecmp(pCfg->charset, pMnode->cfg.charset)) { - mError("\"charset\"[%s - %s] cfg parameters inconsistent.", pCfg->charset, pMnode->cfg.charset); + mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, pMnode->cfg.charset); return DND_REASON_CHARSET_NOT_MATCH; } @@ -287,14 +281,12 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { pStatus->sver = htonl(pStatus->sver); pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->clusterId = htonl(pStatus->clusterId); - pStatus->rebootTime = htonl(pStatus->rebootTime); + pStatus->rebootTime = htobe64(pStatus->rebootTime); pStatus->numOfCores = htons(pStatus->numOfCores); pStatus->numOfSupportMnodes = htons(pStatus->numOfSupportMnodes); pStatus->numOfSupportVnodes = htons(pStatus->numOfSupportVnodes); pStatus->numOfSupportQnodes = htons(pStatus->numOfSupportQnodes); - pStatus->clusterCfg.statusInterval = htonl(pStatus->clusterCfg.statusInterval); - pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(pStatus->clusterCfg.mnodeEqualVnodeNum); pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime); } @@ -308,7 +300,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp); if (pDnode == NULL) { mDebug("dnode:%s, not created yet", pStatus->dnodeEp); - return TSDB_CODE_MND_DNODE_NOT_EXIST; + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; } } else { pDnode = mndAcquireDnode(pMnode, pStatus->dnodeId); @@ -319,7 +312,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp); mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_MND_DNODE_NOT_EXIST; + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; } } @@ -329,7 +323,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } mndReleaseDnode(pMnode, pDnode); mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); - return TSDB_CODE_MND_INVALID_MSG_VERSION; + terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; + return -1; } if (pStatus->dnodeId == 0) { @@ -341,7 +336,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } mError("dnode:%d, clusterId %d not match exist %d", pDnode->id, pStatus->clusterId, pMnode->clusterId); mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_MND_INVALID_CLUSTER_ID; + terrno != TSDB_CODE_MND_INVALID_CLUSTER_ID; + return -1; } else { pDnode->accessTimes++; mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); @@ -355,7 +351,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode->offlineReason = ret; mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[ret]); mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT; + terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG; + return -1; } mInfo("dnode:%d, from offline to online", pDnode->id); @@ -366,6 +363,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode->numOfSupportMnodes = pStatus->numOfSupportMnodes; pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes; pDnode->numOfSupportQnodes = pStatus->numOfSupportQnodes; + pDnode->lastAccessTime = taosGetTimestampMs(); pDnode->status = DND_STATUS_READY; int32_t numOfEps = mndGetDnodeSize(pMnode); @@ -373,7 +371,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { SStatusRsp *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); @@ -390,13 +389,13 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { SDnodeObj dnodeObj = {0}; - dnodeObj.id = 1; // todo + dnodeObj.id = id++; dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.updateTime = dnodeObj.createdTime; taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port); if (dnodeObj.fqdn[0] == 0 || dnodeObj.port <= 0) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_MND_INVALID_DNODE_EP; mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); return terrno; } @@ -449,7 +448,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { mDebug("dnode:%s, start to create", pCreate->ep); if (pCreate->ep[0] == 0) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_MND_INVALID_DNODE_EP; mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); return -1; } @@ -457,7 +456,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, pCreate->ep); if (pDnode != NULL) { mError("dnode:%d, already exist", pDnode->id); - sdbRelease(pMnode->pSdb, pDnode); + mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; return -1; } @@ -478,7 +477,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); return -1; } - mDebug("trans:%d, used to drop user:%d", pTrans->id, pDnode->id); + mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); SSdbRaw *pRedoRaw = mndDnodeActionEncode(pDnode); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { @@ -522,26 +521,26 @@ static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { mDebug("dnode:%d, start to drop", pDrop->dnodeId); if (pDrop->dnodeId <= 0) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_MND_INVALID_DNODE_ID; mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDrop->dnodeId); if (pDnode == NULL) { - mError("dnode:%d, not exist", pDrop->dnodeId); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } int32_t code = mndDropDnode(pMnode, pMsg, pDnode); - if (code != 0) { + mndReleaseDnode(pMnode, pDnode); mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } - sdbRelease(pMnode->pSdb, pDnode); + mndReleaseDnode(pMnode, pDnode); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -553,7 +552,7 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCfg->dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - mError("dnode:%d, failed to cfg since %s ", pCfg->dnodeId, terrstr()); + mError("dnode:%d, failed to config since %s ", pCfg->dnodeId, terrstr()); return -1; } @@ -562,17 +561,22 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { SCfgDnodeMsg *pCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg)); pCfgDnode->dnodeId = htonl(pCfg->dnodeId); - memcpy(pCfgDnode->config, pCfg->config, 128); + memcpy(pCfgDnode->config, pCfg->config, TSDB_DNODE_CONFIG_LEN); - SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN, .pCont = pCfgDnode, .contLen = sizeof(SCfgDnodeMsg)}; + SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN, + .pCont = pCfgDnode, + .contLen = sizeof(SCfgDnodeMsg), + .ahandle = pMsg->rpcMsg.ahandle}; - mInfo("dnode:%d, is configured", pCfg->dnodeId); + mInfo("dnode:%d, app:%p config:%s req send to dnode", pCfg->dnodeId, rpcMsg.ahandle, pCfg->config); mndSendMsgToDnode(pMnode, &epSet, &rpcMsg); return 0; } -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { mInfo("cfg dnode rsp is received"); } +static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { + mInfo("app:%p config rsp from dnode", pMsg->rpcMsg.ahandle); +} static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { int32_t cols = 0; @@ -600,8 +604,7 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->numOfRows = TSDB_CONFIG_NUMBER; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pIter = NULL; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -676,7 +679,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "cores"); + strcpy(pSchema[cols].name, "max vnodes"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -708,7 +711,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->numOfRows = sdbGetSize(pSdb, SDB_DNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -740,7 +743,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDnode->numOfCores; + *(int16_t *)pWrite = pDnode->numOfSupportVnodes; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -753,7 +756,11 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); + if (pDnode->status == DND_STATUS_READY) { + STR_TO_VARSTR(pWrite, ""); + } else { + STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); + } cols++; numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index d2d538b97322207c49400c46b83b3fba327a0e58..e407b271fdfa1cd67b71193947a44bda3b735f4b 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -429,7 +429,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, "show funcs"); + strcpy(pMeta->tbFname, "show funcs"); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 8a05eb02e85f1e69b5bd1c91e7b464daab22874d..6278e3ffef689d1884617450cafccec4a6eb6c74 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -405,7 +405,7 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->numOfRows = sdbGetSize(pSdb, SDB_MNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index b4298797895718b5484f5837e72340811594e43d..59d84e5760206d8a331c23ab3a99dd3490d301e5 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -29,6 +29,7 @@ typedef struct { char user[TSDB_USER_LEN]; char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc int32_t pid; // pid of app that invokes taosc + int64_t appStartTime; // app start time int32_t id; int8_t killed; int8_t align; @@ -44,7 +45,7 @@ typedef struct { SQueryDesc *pQueries; } SConnObj; -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app); +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); @@ -102,13 +103,14 @@ void mndCleanupProfile(SMnode *pMnode) { } } -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app) { +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); SConnObj connObj = {.pid = pid, + .appStartTime = startTime, .id = connId, .killed = 0, .port = port, @@ -195,6 +197,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SConnectMsg *pReq = pMsg->rpcMsg.pCont; pReq->pid = htonl(pReq->pid); + pReq->startTime = htobe64(pReq->startTime); SRpcConnInfo info = {0}; if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { @@ -216,7 +219,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); } - SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, pReq->startTime); if (pConn == NULL) { mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); return -1; @@ -233,9 +236,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser != NULL) { pRsp->acctId = htonl(pUser->acctId); - pRsp->superAuth = pUser->superAuth; - pRsp->readAuth = pUser->readAuth; - pRsp->writeAuth = pUser->writeAuth; + pRsp->superUser = pUser->superUser; mndReleaseUser(pMnode, pUser); } @@ -246,7 +247,8 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { pMsg->contLen = sizeof(SConnectRsp); pMsg->pCont = pRsp; - mDebug("user:%s, login from %s, conn:%d", info.user, ip, pConn->id); + + mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pReq->app); return 0; } @@ -301,7 +303,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId); if (pConn == NULL) { - pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, 0); if (pConn == NULL) { mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); return -1; @@ -368,7 +370,7 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -399,7 +401,7 @@ static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -430,7 +432,7 @@ static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -459,7 +461,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -587,7 +589,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -803,7 +805,7 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index cfecaf27751e0b1ad6ed8989526c1916a6179098..c2b2acfa0cd8dc86221e61b93225a465b8a09ca5 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "mndShow.h" +#define SHOW_STEP_SIZE 100 + static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId); @@ -211,7 +213,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { } /* return no more than 100 tables in one round trip */ - if (rowsToRead > 100) rowsToRead = 100; + if (rowsToRead > SHOW_STEP_SIZE) rowsToRead = SHOW_STEP_SIZE; /* * the actual number of table may be larger than the value of pShow->numOfRows, if a query is @@ -220,7 +222,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (rowsToRead < 0) rowsToRead = 0; size = pShow->rowSize * rowsToRead; - size += 100; + size += SHOW_STEP_SIZE; SRetrieveTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { mndReleaseShowObj(pShow, false); @@ -270,7 +272,7 @@ char *mndShowStr(int32_t showType) { return "show mnodes"; case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; - case TSDB_MGMT_TABLE_STABLE: + case TSDB_MGMT_TABLE_STB: return "show stables"; case TSDB_MGMT_TABLE_MODULE: return "show modules"; diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStable.c deleted file mode 100644 index b57b05f29945be87f8afa763db5ffc9a3b1d4006..0000000000000000000000000000000000000000 --- a/source/dnode/mnode/impl/src/mndStable.c +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "mndStable.h" -#include "mndDnode.h" -#include "mndMnode.h" -#include "mndShow.h" -#include "mndTrans.h" -#include "mndUser.h" -#include "mndDb.h" -#include "tname.h" - -#define TSDB_STABLE_VER_NUM 1 -#define TSDB_STABLE_RESERVE_SIZE 64 - -static SSdbRaw *mndStableActionEncode(SStableObj *pStb); -static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw); -static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb); -static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb); -static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb); -static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg); -static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg); -static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); -static void mndCancelGetNextStable(SMnode *pMnode, void *pIter); - -int32_t mndInitStable(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_STABLE, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndStableActionEncode, - .decodeFp = (SdbDecodeFp)mndStableActionDecode, - .insertFp = (SdbInsertFp)mndStableActionInsert, - .updateFp = (SdbUpdateFp)mndStableActionUpdate, - .deleteFp = (SdbDeleteFp)mndStableActionDelete}; - - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE, mndProcessCreateStableMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE, mndProcessAlterStableMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE, mndProcessDropStableMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP, mndProcessCreateStableInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP, mndProcessAlterStableInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE_IN_RSP, mndProcessDropStableInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStableMetaMsg); - - mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndGetStableMeta); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndRetrieveStables); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndCancelGetNextStable); - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupStable(SMnode *pMnode) {} - -static SSdbRaw *mndStableActionEncode(SStableObj *pStb) { - int32_t size = sizeof(SStableObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); - SSdbRaw *pRaw = sdbAllocRaw(SDB_STABLE, TSDB_STABLE_VER_NUM, size); - if (pRaw == NULL) return NULL; - - int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) - SDB_SET_INT64(pRaw, dataPos, pStb->createdTime) - SDB_SET_INT64(pRaw, dataPos, pStb->updateTime) - SDB_SET_INT64(pRaw, dataPos, pStb->uid) - SDB_SET_INT64(pRaw, dataPos, pStb->version) - SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns) - SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags) - - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { - SSchema *pSchema = &pStb->columnSchema[i]; - SDB_SET_INT8(pRaw, dataPos, pSchema->type); - SDB_SET_INT32(pRaw, dataPos, pSchema->colId); - SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); - SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); - } - - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pStb->tagSchema[i]; - SDB_SET_INT8(pRaw, dataPos, pSchema->type); - SDB_SET_INT32(pRaw, dataPos, pSchema->colId); - SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); - SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); - } - - SDB_SET_RESERVE(pRaw, dataPos, TSDB_STABLE_RESERVE_SIZE) - SDB_SET_DATALEN(pRaw, dataPos); - - return pRaw; -} - -static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { - int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - - if (sver != TSDB_STABLE_VER_NUM) { - mError("failed to decode stable since %s", terrstr()); - terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - return NULL; - } - - int32_t size = sizeof(SStableObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); - SSdbRow *pRow = sdbAllocRow(size); - SStableObj *pStb = sdbGetRowObj(pRow); - if (pStb == NULL) return NULL; - - int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) - SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime) - SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime) - SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid) - SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version) - SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns) - SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags) - - pStb->columnSchema = calloc(pStb->numOfColumns, sizeof(SSchema)); - pStb->tagSchema = calloc(pStb->numOfTags, sizeof(SSchema)); - - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { - SSchema *pSchema = &pStb->columnSchema[i]; - SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); - SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); - SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); - SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); - } - - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pStb->tagSchema[i]; - SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); - SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); - SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); - SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); - } - - SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_STABLE_RESERVE_SIZE) - - return pRow; -} - -static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb) { - mTrace("stable:%s, perform insert action", pStb->name); - return 0; -} - -static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb) { - mTrace("stable:%s, perform delete action", pStb->name); - return 0; -} - -static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb) { - mTrace("stable:%s, perform update action", pOldStb->name); - atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime); - atomic_exchange_32(&pOldStb->version, pNewStb->version); - - taosWLockLatch(&pOldStb->lock); - int32_t numOfTags = pNewStb->numOfTags; - int32_t tagSize = numOfTags * sizeof(SSchema); - int32_t numOfColumns = pNewStb->numOfColumns; - int32_t columnSize = numOfColumns * sizeof(SSchema); - - if (pOldStb->numOfTags < numOfTags) { - pOldStb->tagSchema = malloc(tagSize); - } - if (pOldStb->numOfColumns < numOfColumns) { - pOldStb->columnSchema = malloc(columnSize); - } - - memcpy(pOldStb->tagSchema, pNewStb->tagSchema, tagSize); - memcpy(pOldStb->columnSchema, pNewStb->columnSchema, columnSize); - taosWUnLockLatch(&pOldStb->lock); - return 0; -} - -SStableObj *mndAcquireStb(SMnode *pMnode, char *stbName) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_STABLE, stbName); -} - -void mndReleaseStb(SMnode *pMnode, SStableObj *pStb) { - SSdb *pSdb = pMnode->pSdb; - sdbRelease(pSdb, pStb); -} - -static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg) { return 0; } - -static SDbObj *mndGetDbByStbName(SMnode *pMnode, char *stbName) { - SName name = {0}; - tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - - char db[TSDB_TABLE_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, db); - - return mndAcquireDb(pMnode, db); -} - -static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SStableInfoMsg *pInfo = pMsg->rpcMsg.pCont; - - mDebug("stable:%s, start to retrieve meta", pInfo->name); - - SDbObj *pDb = mndGetDbByStbName(pMnode, pInfo->name); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stable:%s, failed to retrieve meta since %s", pInfo->name, terrstr()); - return -1; - } - - SStableObj *pStb = mndAcquireStb(pMnode, pInfo->name); - if (pStb == NULL) { - mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_MND_INVALID_TABLE_NAME; - mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr()); - return -1; - } - - int32_t contLen = sizeof(STableMetaMsg) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); - STableMetaMsg *pMeta = rpcMallocCont(contLen); - if (pMeta == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr()); - return -1; - } - - memcpy(pMeta->stableFname, pStb->name, TSDB_TABLE_FNAME_LEN); - pMeta->numOfTags = htonl(pStb->numOfTags); - pMeta->numOfColumns = htonl(pStb->numOfColumns); - pMeta->precision = pDb->cfg.precision; - pMeta->tableType = TSDB_SUPER_TABLE; - pMeta->update = pDb->cfg.update; - pMeta->sversion = htonl(pStb->version); - pMeta->suid = htonl(pStb->uid); - - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { - SSchema *pSchema = &pMeta->pSchema[i]; - SSchema *pColumn = &pStb->columnSchema[i]; - memcpy(pSchema->name, pColumn->name, TSDB_COL_NAME_LEN); - pSchema->type = pColumn->type; - pSchema->colId = htonl(pColumn->colId); - pSchema->bytes = htonl(pColumn->bytes); - } - - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pMeta->pSchema[i + pStb->numOfColumns]; - SSchema *pTag = &pStb->tagSchema[i]; - memcpy(pSchema->name, pTag->name, TSDB_COL_NAME_LEN); - pSchema->type = pTag->type; - pSchema->colId = htons(pTag->colId); - pSchema->bytes = htonl(pTag->bytes); - } - - pMsg->pCont = pMeta; - pMsg->contLen = contLen; - - mDebug("stable:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags); - return 0; -} - -static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfStables) { - SSdb *pSdb = pMnode->pSdb; - - SDbObj *pDb = mndAcquireDb(pMnode, dbName); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; - } - - int32_t numOfStables = 0; - void *pIter = NULL; - while (1) { - SStableObj *pStb = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb); - if (pIter == NULL) break; - - if (strcmp(pStb->db, dbName) == 0) { - numOfStables++; - } - - sdbRelease(pSdb, pStb); - } - - *pNumOfStables = numOfStables; - return 0; -} - -static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; - SSdb *pSdb = pMnode->pSdb; - - if (mndGetNumOfStables(pMnode, pShow->db, &pShow->numOfRows) != 0) { - return -1; - } - - int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; - - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "columns"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "tags"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htons(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) { - pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - } - - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); - - return 0; -} - -static void mnodeExtractTableName(char* tableId, char* name) { - int pos = -1; - int num = 0; - for (pos = 0; tableId[pos] != 0; ++pos) { - if (tableId[pos] == '.') num++; - if (num == 2) break; - } - - if (num == 2) { - strcpy(name, tableId + pos + 1); - } -} - -static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - SStableObj *pStb = NULL; - int32_t cols = 0; - char *pWrite; - char prefix[64] = {0}; - - tstrncpy(prefix, pShow->db, 64); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = (int32_t)strlen(prefix); - - while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_STABLE, pShow->pIter, (void **)&pStb); - if (pShow->pIter == NULL) break; - - if (strncmp(pStb->name, prefix, prefixLen) != 0) { - sdbRelease(pSdb, pStb); - continue; - } - - cols = 0; - - char stableName[TSDB_TABLE_FNAME_LEN] = {0}; - memcpy(stableName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, stableName); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pStb->createdTime; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStb->numOfColumns; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStb->numOfTags; - cols++; - - numOfRows++; - sdbRelease(pSdb, pStb); - } - - pShow->numOfReads += numOfRows; - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); - return numOfRows; -} - -static void mndCancelGetNextStable(SMnode *pMnode, void *pIter) { - SSdb *pSdb = pMnode->pSdb; - sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c new file mode 100644 index 0000000000000000000000000000000000000000..c3afbf37c8eb2c7e453ad816a55d39cd344d8163 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -0,0 +1,675 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndStb.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "tname.h" + +#define TSDB_STB_VER_NUM 1 +#define TSDB_STB_RESERVE_SIZE 64 + +static SSdbRaw *mndStbActionEncode(SStbObj *pStb); +static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); +static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); +static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); +static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb); +static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg); +static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); + +int32_t mndInitStb(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_STB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStbActionEncode, + .decodeFp = (SdbDecodeFp)mndStbActionDecode, + .insertFp = (SdbInsertFp)mndStbActionInsert, + .updateFp = (SdbUpdateFp)mndStbActionUpdate, + .deleteFp = (SdbDeleteFp)mndStbActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STB, mndProcessCreateStbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STB, mndProcessAlterStbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STB, mndProcessDropStbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STB_IN_RSP, mndProcessCreateStbInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STB_IN_RSP, mndProcessAlterStbInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STB_IN_RSP, mndProcessDropStbInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStbMetaMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STB, mndGetStbMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STB, mndCancelGetNextStb); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupStb(SMnode *pMnode) {} + +static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_SET_INT64(pRaw, dataPos, pStb->createdTime) + SDB_SET_INT64(pRaw, dataPos, pStb->updateTime) + SDB_SET_INT64(pRaw, dataPos, pStb->uid) + SDB_SET_INT64(pRaw, dataPos, pStb->version) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags) + + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pStb->pSchema[i]; + SDB_SET_INT8(pRaw, dataPos, pSchema->type); + SDB_SET_INT32(pRaw, dataPos, pSchema->colId); + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_STB_VER_NUM) { + mError("failed to decode stable since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + int32_t size = sizeof(SStbObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); + SSdbRow *pRow = sdbAllocRow(size); + SStbObj *pStb = sdbGetRowObj(pRow); + if (pStb == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags) + + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + pStb->pSchema = calloc(totalCols, sizeof(SSchema)); + + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pStb->pSchema[i]; + SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); + SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_STB_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { + mTrace("stb:%s, perform insert action", pStb->name); + return 0; +} + +static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { + mTrace("stb:%s, perform delete action", pStb->name); + return 0; +} + +static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb) { + mTrace("stb:%s, perform update action", pOldStb->name); + atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime); + atomic_exchange_32(&pOldStb->version, pNewStb->version); + + taosWLockLatch(&pOldStb->lock); + int32_t totalCols = pNewStb->numOfTags + pNewStb->numOfColumns; + int32_t totalSize = totalCols * sizeof(SSchema); + + if (pOldStb->numOfTags + pOldStb->numOfColumns < totalCols) { + pOldStb->pSchema = malloc(totalSize); + } + + memcpy(pOldStb->pSchema, pNewStb->pSchema, totalSize); + taosWUnLockLatch(&pOldStb->lock); + return 0; +} + +SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_STB, stbName); +} + +void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pStb); +} + +static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) { + SName name = {0}; + tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) { + pCreate->numOfColumns = htonl(pCreate->numOfColumns); + pCreate->numOfTags = htonl(pCreate->numOfTags); + int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags; + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pCreate->pSchema[i]; + pSchema->colId = htonl(pSchema->colId); + pSchema->bytes = htonl(pSchema->bytes); + } + + if (pCreate->igExists < 0 || pCreate->igExists > 1) { + terrno = TSDB_CODE_MND_STB_INVALID_IGEXIST; + return -1; + } + + if (pCreate->numOfColumns < TSDB_MIN_COLUMNS || pCreate->numOfColumns > TSDB_MAX_COLUMNS) { + terrno = TSDB_CODE_MND_STB_INVALID_COLS_NUM; + return -1; + } + + if (pCreate->numOfTags <= 0 || pCreate->numOfTags > TSDB_MAX_TAGS) { + terrno = TSDB_CODE_MND_STB_INVALID_TAGS_NUM; + return -1; + } + + int32_t maxColId = (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS); + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pCreate->pSchema[i]; + if (pSchema->type <= 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE; + return -1; + } + if (pSchema->colId < 0 || pSchema->colId >= maxColId) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_ID; + return -1; + } + if (pSchema->bytes <= 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES; + return -1; + } + if (pSchema->name[0] == 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME; + return -1; + } + } + + return 0; +} + +static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCreate, SDbObj *pDb) { + SStbObj stbObj = {0}; + tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); + tstrncpy(stbObj.db, pDb->name, TSDB_FULL_DB_NAME_LEN); + stbObj.createdTime = taosGetTimestampMs(); + stbObj.updateTime = stbObj.createdTime; + stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + stbObj.version = 1; + stbObj.numOfColumns = pCreate->numOfColumns; + stbObj.numOfTags = pCreate->numOfTags; + + int32_t totalCols = stbObj.numOfColumns + stbObj.numOfTags; + int32_t totalSize = totalCols * sizeof(SSchema); + stbObj.pSchema = malloc(totalSize); + if (stbObj.pSchema == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(stbObj.pSchema, pCreate->pSchema, totalSize); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); + + SSdbRaw *pRedoRaw = mndStbActionEncode(&stbObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndStbActionEncode(&stbObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateStbMsg *pCreate = pMsg->rpcMsg.pCont; + + mDebug("stb:%s, start to create", pCreate->name); + + if (mndCheckStbMsg(pCreate) != 0) { + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); + if (pStb != NULL) { + sdbRelease(pMnode->pSdb, pStb); + if (pCreate->igExists) { + mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); + return 0; + } else { + terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; + mError("db:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + } + + SDbObj *pDb = mndAcquireDbByStb(pMnode, pCreate->name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + int32_t code = mndCreateStb(pMnode, pMsg, pCreate, pDb); + mndReleaseDb(pMnode, pDb); + + if (code != 0) { + terrno = code; + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { + SSchema *pSchema = &pAlter->schema; + pSchema->colId = htonl(pSchema->colId); + pSchema->bytes = htonl(pSchema->bytes); + + if (pSchema->type <= 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE; + return -1; + } + if (pSchema->colId < 0 || pSchema->colId >= (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_ID; + return -1; + } + if (pSchema->bytes <= 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES; + return -1; + } + if (pSchema->name[0] == 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME; + return -1; + } + + return 0; +} + +static int32_t mndUpdateStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pOldStb, SStbObj *pNewStb) { return 0; } + +static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SAlterStbMsg *pAlter = pMsg->rpcMsg.pCont; + + mDebug("stb:%s, start to alter", pAlter->name); + + if (mndCheckAlterStbMsg(pAlter) != 0) { + mError("stb:%s, failed to alter since %s", pAlter->name, terrstr()); + return -1; + } + + SStbObj *pStb = mndAcquireStb(pMnode, pAlter->name); + if (pStb == NULL) { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + mError("stb:%s, failed to alter since %s", pAlter->name, terrstr()); + return -1; + } + + SStbObj stbObj = {0}; + memcpy(&stbObj, pStb, sizeof(SStbObj)); + + int32_t code = mndUpdateStb(pMnode, pMsg, pStb, &stbObj); + mndReleaseStb(pMnode, pStb); + + if (code != 0) { + mError("stb:%s, failed to alter since %s", pAlter->name, tstrerror(code)); + return code; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("stb:%s, failed to drop since %s", pStb->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); + + SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropStbMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("stb:%s, start to drop", pDrop->name); + + SStbObj *pStb = mndAcquireStb(pMnode, pDrop->name); + if (pStb == NULL) { + if (pDrop->igNotExists) { + mDebug("stb:%s, not exist, ignore not exist is set", pDrop->name); + return 0; + } else { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + } + + int32_t code = mndDropStb(pMnode, pMsg, pStb); + mndReleaseStb(pMnode, pStb); + + if (code != 0) { + terrno = code; + mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SStbInfoMsg *pInfo = pMsg->rpcMsg.pCont; + + mDebug("stb:%s, start to retrieve meta", pInfo->name); + + SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stb:%s, failed to retrieve meta since %s", pInfo->name, terrstr()); + return -1; + } + + SStbObj *pStb = mndAcquireStb(pMnode, pInfo->name); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_MND_INVALID_TABLE_NAME; + mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); + return -1; + } + + int32_t contLen = sizeof(STableMetaMsg) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + STableMetaMsg *pMeta = rpcMallocCont(contLen); + if (pMeta == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); + return -1; + } + + memcpy(pMeta->stbFname, pStb->name, TSDB_TABLE_FNAME_LEN); + pMeta->numOfTags = htonl(pStb->numOfTags); + pMeta->numOfColumns = htonl(pStb->numOfColumns); + pMeta->precision = pDb->cfg.precision; + pMeta->tableType = TSDB_SUPER_TABLE; + pMeta->update = pDb->cfg.update; + pMeta->sversion = htonl(pStb->version); + pMeta->suid = htonl(pStb->uid); + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pMeta->pSchema[i]; + SSchema *pSrcSchema = &pStb->pSchema[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + + pMsg->pCont = pMeta; + pMsg->contLen = contLen; + + mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags); + return 0; +} + +static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { + SSdb *pSdb = pMnode->pSdb; + + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + int32_t numOfStbs = 0; + void *pIter = NULL; + while (1) { + SStbObj *pStb = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb); + if (pIter == NULL) break; + + if (strcmp(pStb->db, dbName) == 0) { + numOfStbs++; + } + + sdbRelease(pSdb, pStb); + } + + *pNumOfStbs = numOfStbs; + return 0; +} + +static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + if (mndGetNumOfStbs(pMnode, pShow->db, &pShow->numOfRows) != 0) { + return -1; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "columns"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "tags"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static void mnodeExtractTableName(char *tableId, char *name) { + int pos = -1; + int num = 0; + for (pos = 0; tableId[pos] != 0; ++pos) { + if (tableId[pos] == '.') num++; + if (num == 2) break; + } + + if (num == 2) { + strcpy(name, tableId + pos + 1); + } +} + +static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SStbObj *pStb = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[64] = {0}; + + tstrncpy(prefix, pShow->db, 64); + strcat(prefix, TS_PATH_DELIMITER); + int32_t prefixLen = (int32_t)strlen(prefix); + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); + if (pShow->pIter == NULL) break; + + if (strncmp(pStb->name, prefix, prefixLen) != 0) { + sdbRelease(pSdb, pStb); + continue; + } + + cols = 0; + + char stbName[TSDB_TABLE_FNAME_LEN] = {0}; + memcpy(stbName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, stbName); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pStb->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pStb->numOfColumns; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pStb->numOfTags; + cols++; + + numOfRows++; + sdbRelease(pSdb, pStb); + } + + pShow->numOfReads += numOfRows; + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index fc1f441a9040f92d57cd3da2a1f576beaa8ebdba..2111047219a850a289c82cf5a1246c136b0c8cc3 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -65,11 +65,9 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.readAuth = 1; - userObj.writeAuth = 1; if (strcmp(user, TSDB_DEFAULT_USER) == 0) { - userObj.superAuth = 1; + userObj.superUser = 1; } SSdbRaw *pRaw = mndUserActionEncode(&userObj); @@ -85,9 +83,11 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { return -1; } +#if 0 if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { return -1; } +#endif return 0; } @@ -98,13 +98,11 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t dataPos = 0; SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN) - SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_KEY_LEN) + SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN) SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) - SDB_SET_INT8(pRaw, dataPos, pUser->superAuth) - SDB_SET_INT8(pRaw, dataPos, pUser->readAuth) - SDB_SET_INT8(pRaw, dataPos, pUser->writeAuth) + SDB_SET_INT8(pRaw, dataPos, pUser->superUser) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -126,13 +124,11 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->user, TSDB_USER_LEN) - SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->pass, TSDB_KEY_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->pass, TSDB_PASSWORD_LEN) SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->acct, TSDB_USER_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superAuth) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->readAuth) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->writeAuth) + SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superUser) return pRow; } @@ -171,13 +167,11 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) { mTrace("user:%s, perform update action", pOldUser->user); memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN); - memcpy(pOldUser->pass, pNewUser->pass, TSDB_KEY_LEN); + memcpy(pOldUser->pass, pNewUser->pass, TSDB_PASSWORD_LEN); memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN); pOldUser->createdTime = pNewUser->createdTime; pOldUser->updateTime = pNewUser->updateTime; - pOldUser->superAuth = pNewUser->superAuth; - pOldUser->readAuth = pNewUser->readAuth; - pOldUser->writeAuth = pNewUser->writeAuth; + pOldUser->superUser = pNewUser->superUser; return 0; } @@ -198,9 +192,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superAuth = 0; - userObj.readAuth = 1; - userObj.writeAuth = 1; + userObj.superUser = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { @@ -469,7 +461,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -489,7 +481,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -513,14 +505,11 @@ static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pUser->superAuth) { + if (pUser->superUser) { const char *src = "super"; STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); - } else if (pUser->writeAuth) { - const char *src = "writable"; - STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); } else { - const char *src = "readable"; + const char *src = "normal"; STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); } cols++; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a0af5f7f5e87e987793c6b19e23ce40be4b4bab6..44d13d1fb498cb41a87fb69ab072d2e26be9895e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -45,7 +45,7 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_VGROUP, - .keyType = SDB_KEY_BINARY, + .keyType = SDB_KEY_INT32, .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, .insertFp = (SdbInsertFp)mndVgroupActionInsert, @@ -238,7 +238,7 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -337,7 +337,7 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->replica = dnodeId; pShow->numOfRows = mndGetVnodesNum(pMnode, dnodeId); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3e0ac746fca48ce793969552b23ed788d518d9b8..e5434786083bf74a1546a1f1585b3fa2a17c8d79 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -24,7 +24,7 @@ #include "mndMnode.h" #include "mndProfile.h" #include "mndShow.h" -#include "mndStable.h" +#include "mndStb.h" #include "mndSync.h" #include "mndTelem.h" #include "mndTrans.h" @@ -131,7 +131,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-stable", mndInitStable, mndCleanupStable) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (pMnode->clusterId <= 0) { if (mndAllocStep(pMnode, "mnode-sdb-deploy", mndDeploySdb, NULL) != 0) return -1; @@ -205,7 +205,6 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->cfg.sver = pOption->cfg.sver; pMnode->cfg.enableTelem = pOption->cfg.enableTelem; pMnode->cfg.statusInterval = pOption->cfg.statusInterval; - pMnode->cfg.mnodeEqualVnodeNum = pOption->cfg.mnodeEqualVnodeNum; pMnode->cfg.shellActivityTimer = pOption->cfg.shellActivityTimer; pMnode->cfg.timezone = strdup(pOption->cfg.timezone); pMnode->cfg.locale = strdup(pOption->cfg.locale); @@ -215,7 +214,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { if (pMnode->sendMsgToDnodeFp == NULL || pMnode->sendMsgToMnodeFp == NULL || pMnode->sendRedirectMsgFp == NULL || pMnode->putMsgToApplyMsgFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 || - pMnode->cfg.statusInterval < 1 || pOption->cfg.mnodeEqualVnodeNum < 0) { + pMnode->cfg.statusInterval < 1) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } @@ -337,8 +336,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { } SRpcConnInfo connInfo = {0}; - if (rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { - mndCleanupMsg(pMsg); + if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { + taosFreeQitem(pMsg); terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; mError("RPC:%p, app:%p failed to create msg since %s", pRpcMsg->handle, pRpcMsg->ahandle, terrstr()); return NULL; @@ -355,6 +354,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { void mndCleanupMsg(SMnodeMsg *pMsg) { mTrace("msg:%p, app:%p is destroyed, RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle); + rpcFreeCont(pMsg->rpcMsg.pCont); + pMsg->rpcMsg.pCont = NULL; taosFreeQitem(pMsg); } @@ -368,7 +369,7 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { int32_t code = 0; int32_t msgType = pMsg->rpcMsg.msgType; void *ahandle = pMsg->rpcMsg.ahandle; - bool isReq = (msgType % 2 == 1); + bool isReq = (msgType & 1U); mTrace("msg:%p, app:%p type:%s will be processed", pMsg, ahandle, taosMsg[msgType]); @@ -430,3 +431,10 @@ void mndProcessWriteMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } void mndProcessSyncMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } void mndProcessApplyMsg(SMnodeMsg *pMsg) {} + +uint64_t mndGenerateUid(char *name, int32_t len) { + int64_t us = taosGetTimestampUs(); + int32_t hashval = MurmurHash3_32(name, len); + uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; + return x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); +} \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 1d4888c2eb37a418e66b994a63510d7f9ba79a72..ccff5b6c82c773df7277e9c13c12f22f69ca2171 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -52,10 +52,10 @@ SSdb *sdbInit(SSdbOpt *pOption) { void sdbCleanup(SSdb *pSdb) { mDebug("start to cleanup sdb"); - if (pSdb->curVer != pSdb->lastCommitVer) { + // if (pSdb->curVer != pSdb->lastCommitVer) { mDebug("write sdb file for curVer:% " PRId64 " and lastVer:%" PRId64, pSdb->curVer, pSdb->lastCommitVer); sdbWriteFile(pSdb); - } + // } if (pSdb->currDir != NULL) { tfree(pSdb->currDir); diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index d6d267c4d441483c92049e5bdf1549ac4459b718..6972605afd9c1dd93b7fd0b3198c3837b5351f88 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -18,6 +18,6 @@ target_link_libraries( ) # test -#if(${BUILD_TEST}) -# add_subdirectory(test) -#endif(${BUILD_TEST}) \ No newline at end of file +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index d64dc93847ba02f95851df3b34fa708f40705cbb..bfc4de9e1261923097ebc1c349bd77c7e42bfe2e 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -28,6 +28,7 @@ typedef struct SVBufPool SVBufPool; int vnodeOpenBufPool(SVnode *pVnode); void vnodeCloseBufPool(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); +bool vnodeBufPoolIsFull(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/impl/inc/vnodeCommit.h index c8ff4947aa4fb9b468637d8e582d90abdf6867a9..a60e8feac21fcc6361405fb67247e77d8739d7dd 100644 --- a/source/dnode/vnode/impl/inc/vnodeCommit.h +++ b/source/dnode/vnode/impl/inc/vnodeCommit.h @@ -22,8 +22,8 @@ extern "C" { #endif -bool vnodeShouldCommit(SVnode *pVnode); -int vnodeAsyncCommit(SVnode *pVnode); +#define vnodeShouldCommit vnodeBufPoolIsFull +int vnodeAsyncCommit(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index c92de433c361726373788ab830bddafa00e9222b..e3a3fac6b90839da79c39450669e95ce45e1b640 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -18,15 +18,17 @@ #include "mallocator.h" #include "sync.h" +#include "tcoding.h" +#include "tdlist.h" #include "tlockfree.h" #include "wal.h" -#include "tcoding.h" #include "vnode.h" #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" #include "vnodeFS.h" +#include "vnodeMemAllocator.h" #include "vnodeRequest.h" #include "vnodeStateMgr.h" #include "vnodeSync.h" diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index 9184eb416b07480c98cca2ccaad2df3ada1dd13a..c8c58e9f69d8d7c31b5592b8ad3f52f9ceae4b02 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -16,15 +16,37 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ -#include "mallocator.h" -#include "vnode.h" +#include "os.h" #ifdef __cplusplus extern "C" { #endif -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); -void vnodeDestroyMemAllocator(SMemAllocator *pma); +typedef struct SVArenaNode SVArenaNode; +typedef struct SVMemAllocator SVMemAllocator; + +struct SVArenaNode { + TD_SLIST_NODE(SVArenaNode); + uint64_t size; // current node size + void * ptr; + char data[]; +}; + +struct SVMemAllocator { + TD_DLIST_NODE(SVMemAllocator); + uint64_t capacity; + uint64_t ssize; + uint64_t lsize; + SVArenaNode *pNode; + TD_SLIST(SVArenaNode) nlist; +}; + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); +void vmaDestroy(SVMemAllocator *pVMA); +void vmaReset(SVMemAllocator *pVMA); +void * vmaMalloc(SVMemAllocator *pVMA, uint64_t size); +void vmaFree(SVMemAllocator *pVMA, void *ptr); +bool vmaIsFull(SVMemAllocator *pVMA); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c new file mode 100644 index 0000000000000000000000000000000000000000..99d4781df9ab286bbcfe5eff92e25fbd8b4c86e0 --- /dev/null +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeDef.h" + +static SVArenaNode *vArenaNodeNew(uint64_t capacity); +static void vArenaNodeFree(SVArenaNode *pNode); + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { + SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); + if (pVMA == NULL) { + return NULL; + } + + pVMA->capacity = capacity; + pVMA->ssize = ssize; + pVMA->lsize = lsize; + tSListInit(&(pVMA->nlist)); + + pVMA->pNode = vArenaNodeNew(capacity); + if (pVMA->pNode == NULL) { + free(pVMA); + return NULL; + } + + tSListPush(&(pVMA->nlist), pVMA->pNode); + + return pVMA; +} + +void vmaDestroy(SVMemAllocator *pVMA) { + if (pVMA) { + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); + vArenaNodeFree(pNode); + } + + free(pVMA); + } +} + +void vmaReset(SVMemAllocator *pVMA) { + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); + vArenaNodeFree(pNode); + } + + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + pNode->ptr = pNode->data; +} + +void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + void * ptr; + + if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { + uint64_t capacity = MAX(pVMA->ssize, size); + pNode = vArenaNodeNew(capacity); + if (pNode == NULL) { + // TODO: handle error + return NULL; + } + + tSListPush(&(pVMA->nlist), pNode); + } + + ptr = pNode->ptr; + pNode->ptr = POINTER_SHIFT(ptr, size); + + return ptr; +} + +void vmaFree(SVMemAllocator *pVMA, void *ptr) { + // TODO +} + +bool vmaIsFull(SVMemAllocator *pVMA) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + + return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) || + (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static SVArenaNode *vArenaNodeNew(uint64_t capacity) { + SVArenaNode *pNode = NULL; + + pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity); + if (pNode == NULL) { + return NULL; + } + + pNode->size = capacity; + pNode->ptr = pNode->data; + + return pNode; +} + +static void vArenaNodeFree(SVArenaNode *pNode) { + if (pNode) { + free(pNode); + } +} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 00203ed9b6c31996c4e03740f9634533070aff34..1db15c39906c435f3148d6089924f0652a723ad8 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -19,110 +19,59 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - // buffer pool impl - SList free; - SList incycle; - SListNode *inuse; + TD_DLIST(SVMemAllocator) free; + TD_DLIST(SVMemAllocator) incycle; + SVMemAllocator *inuse; // MAF for submodules - SMemAllocatorFactory maf; + // SMemAllocatorFactory maf; }; -typedef enum { - // Heap allocator - E_V_HEAP_ALLOCATOR = 0, - // Arena allocator - E_V_ARENA_ALLOCATOR -} EVMemAllocatorT; - -typedef struct { - /* TODO */ -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - uint64_t size; - void * ptr; - char data[]; -} SVArenaNode; - -typedef struct { - uint64_t ssize; // step size - uint64_t lsize; // limit size - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - SVnode * pVnode; - SListNode *pNode; -} SVMAWrapper; - -typedef struct { - T_REF_DECLARE() - uint64_t capacity; - EVMemAllocatorT type; - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; - -static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); -static void vBufPoolFreeNode(SListNode *pNode); -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); -static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); -static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); - int vnodeOpenBufPool(SVnode *pVnode) { - uint64_t capacity; - EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; + uint64_t capacity; + // EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { /* TODO */ return -1; } - tdListInit(&(pVnode->pBufPool->free), 0); - tdListInit(&(pVnode->pBufPool->incycle), 0); + tDListInit(&(pVnode->pBufPool->free)); + tDListInit(&(pVnode->pBufPool->incycle)); + pVnode->pBufPool->inuse = NULL; + + // TODO capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; - if (pVnode->config.isHeapAllocator) { - type = E_V_HEAP_ALLOCATOR; - } for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { - SListNode *pNode = vBufPoolNewNode(capacity, type); - if (pNode == NULL) { - vnodeCloseBufPool(pVnode); + SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize); + if (pVMA == NULL) { + // TODO: handle error return -1; } - tdListAppendNode(&(pVnode->pBufPool->free), pNode); + tDListAppend(&(pVnode->pBufPool->free), pVMA); } - pVnode->pBufPool->maf.impl = pVnode; - pVnode->pBufPool->maf.create = vBufPoolCreateMA; - pVnode->pBufPool->maf.destroy = vBufPoolDestroyMA; - return 0; } void vnodeCloseBufPool(SVnode *pVnode) { - SListNode *pNode; if (pVnode->pBufPool) { - // Clear free list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) { - vBufPoolFreeNode(pNode); - } + vmaDestroy(pVnode->pBufPool->inuse); - // Clear incycle list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) { - vBufPoolFreeNode(pNode); + while (true) { + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle)); + if (pVMA == NULL) break; + tDListPop(&(pVnode->pBufPool->incycle), pVMA); + vmaDestroy(pVMA); } - // Free inuse node - if (pVnode->pBufPool->inuse) { - vBufPoolFreeNode(pVnode->pBufPool->inuse); + while (true) { + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free)); + if (pVMA == NULL) break; + tDListPop(&(pVnode->pBufPool->free), pVMA); + vmaDestroy(pVMA); } free(pVnode->pBufPool); @@ -131,48 +80,73 @@ void vnodeCloseBufPool(SVnode *pVnode) { } void *vnodeMalloc(SVnode *pVnode, uint64_t size) { - void *ptr; - - if (pVnode->pBufPool->inuse == NULL) { - SListNode *pNode; - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) == NULL) { - // todo - // tsem_wait(); - ASSERT(0); + SVBufPool *pBufPool = pVnode->pBufPool; + + if (pBufPool->inuse == NULL) { + while (true) { + // TODO: add sem_wait and sem_post + pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free)); + if (pBufPool->inuse) { + tDListPop(&(pBufPool->free), pBufPool->inuse); + break; + } } - - pVnode->pBufPool->inuse = pNode; } - SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - return vBufPoolMalloc(pvma, size); + return vmaMalloc(pBufPool->inuse, size); } -/* ------------------------ STATIC METHODS ------------------------ */ -static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */ - pvaa->ssize = ssize; - pvaa->lsize = lsize; - pvaa->inuse = &pvaa->node; - - pvaa->node.prev = NULL; - pvaa->node.size = capacity; - pvaa->node.ptr = pvaa->node.data; +bool vnodeBufPoolIsFull(SVnode *pVnode) { + if (pVnode->pBufPool->inuse == NULL) return false; + return vmaIsFull(pVnode->pBufPool->inuse); } -static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { /* TODO */ - while (pvaa->inuse != &(pvaa->node)) { - SVArenaNode *pANode = pvaa->inuse; - pvaa->inuse = pANode->prev; - free(pANode); - } -} +#if 0 +typedef enum { + // Heap allocator + E_V_HEAP_ALLOCATOR = 0, + // Arena allocator + E_V_ARENA_ALLOCATOR +} EVMemAllocatorT; + +typedef struct { + /* TODO */ +} SVHeapAllocator; + +typedef struct SVArenaNode { + struct SVArenaNode *prev; + uint64_t size; + void * ptr; + char data[]; +} SVArenaNode; + +typedef struct { + uint64_t ssize; // step size + uint64_t lsize; // limit size + SVArenaNode *inuse; + SVArenaNode node; +} SVArenaAllocator; + +typedef struct { + SVnode * pVnode; + SListNode *pNode; +} SVMAWrapper; + + +static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); +static void vBufPoolFreeNode(SListNode *pNode); +static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); +static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); +static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); + +/* ------------------------ STATIC METHODS ------------------------ */ static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { SListNode * pNode; SVMemAllocator *pvma; uint64_t msize; - uint64_t ssize = 0; // TODO - uint64_t lsize = 0; // TODO + uint64_t ssize = 4096; // TODO + uint64_t lsize = 1024; // TODO msize = sizeof(SListNode) + sizeof(SVMemAllocator); if (type == E_V_ARENA_ALLOCATOR) { @@ -317,4 +291,5 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) { tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); // tsem_post(); todo: sem_post } -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/impl/src/vnodeCfg.c index 01facba88856ae342c207ac5065ca261613458ae..f5bb7e35d258d1791098490983f706b4b32ad5bc 100644 --- a/source/dnode/vnode/impl/src/vnodeCfg.c +++ b/source/dnode/vnode/impl/src/vnodeCfg.c @@ -15,7 +15,7 @@ #include "vnodeDef.h" -const SVnodeCfg defaultVnodeOptions = {0}; /* TODO */ +const SVnodeCfg defaultVnodeOptions = {.wsize = 16 * 1024 * 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index 18a0c6d91dc065b63a0a90cd7d021afd45beabf6..cac7999f59c8fcb70308c64a79f0edcc5fa7a9ac 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -18,8 +18,6 @@ static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); -bool vnodeShouldCommit(SVnode *pVnode) { return false; } - int vnodeAsyncCommit(SVnode *pVnode) { #if 0 if (vnodeStartCommit(pVnode) < 0) { diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index ab33b58858e14f1430deb28f7114374a4862f8fe..9b94b4a361c8274563554c36ae52b6fc36d258f6 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tmacro.h" #include "vnodeDef.h" static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); @@ -20,8 +21,14 @@ static void vnodeFree(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode); +TD_DEF_MOD_INIT_FLAG(vnode); +TD_DEF_MOD_CLEAR_FLAG(vnode); + int vnodeInit() { - // TODO + if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) { + return 0; + } + if (walInit() < 0) { return -1; } @@ -30,6 +37,10 @@ int vnodeInit() { } void vnodeClear() { + if (TD_CHECK_AND_SET_MOD_CLEAR(vnode) == TD_MOD_CLEARD) { + return; + } + walCleanUp(); } diff --git a/source/dnode/vnode/impl/src/vnodeMemAllocator.c b/source/dnode/vnode/impl/src/vnodeMemAllocator.c deleted file mode 100644 index 902014eb4767e5df9cbf67a5ea6f681a4b718361..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/impl/src/vnodeMemAllocator.c +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) { - SMemAllocator *pma = NULL; - /* TODO */ - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -#if 0 -#define VNODE_HEAP_ALLOCATOR 0 -#define VNODE_ARENA_ALLOCATOR 1 - -typedef struct { - uint64_t tsize; - uint64_t used; -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - void * nptr; - char data[]; -} SVArenaNode; - -typedef struct { - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - int8_t type; - uint64_t tsize; - T_REF_DECLARE() - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; - -SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) { - SMemAllocator * pma; - uint64_t msize; - SVMemAllocator *pva; - - msize = sizeof(*pma) + sizeof(SVMemAllocator); - if (type == VNODE_ARENA_ALLOCATOR) { - msize += tsize; - } - - pma = (SMemAllocator *)calloc(1, msize); - if (pma == NULL) { - return NULL; - } - - pma->impl = POINTER_SHIFT(pma, sizeof(*pma)); - pva = (SVMemAllocator *)(pma->impl); - pva->type = type; - pva->tsize = tsize; - - if (type == VNODE_HEAP_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else if (type == VNODE_ARENA_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else { - ASSERT(0); - } - - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeRefMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeUnrefMemAllocator(SMemAllocator *pma) { - // TODO -} - -/* ------------------------ Heap Allocator IMPL ------------------------ */ - -/* ------------------------ Arena Allocator IMPL ------------------------ */ - -#endif \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/CMakeLists.txt b/source/dnode/vnode/impl/test/CMakeLists.txt index 83506a4fdee92379acba148c879514d99a849b5e..e1226331e9649081057701da94f18fb23de6d154 100644 --- a/source/dnode/vnode/impl/test/CMakeLists.txt +++ b/source/dnode/vnode/impl/test/CMakeLists.txt @@ -4,4 +4,9 @@ target_sources(vnodeApiTests PRIVATE "vnodeApiTests.cpp" ) -target_link_libraries(vnodeApiTests vnode gtest gtest_main) \ No newline at end of file +target_link_libraries(vnodeApiTests vnode gtest gtest_main) + +add_test( + NAME vnode_api_tests + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests + ) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index df784181b7e4ba487722a2de93f596271b0ed768..ac2ccbc13290c092b7fb54ebf16984a47d88837b 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -1,3 +1,14 @@ +/** + * @file vnodeApiTests.cpp + * @author hzcheng (hzcheng@taosdata.com) + * @brief VNODE module API tests + * @version 0.1 + * @date 2021-12-13 + * + * @copyright Copyright (c) 2021 + * + */ + #include #include diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3837e5408a1e95a875baec41352fe609660bfb1e..f4a4b01dab19a022722727a6f95a07d51a169f1e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -283,8 +283,12 @@ int32_t catalogGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, cons return TSDB_CODE_SUCCESS; } +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta) { -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SCatalogRsp* pRsp) { +} + + +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { if (NULL == pCatalog || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { return TSDB_CODE_CTG_INVALID_INPUT; } diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index eb288e0aa2b021c080b1faf156f7960ca1e3b0ca..a1d4962e8be815847ca1a25689588dcb157c318f 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -26,10 +26,24 @@ extern "C" { #include "index_fst_counting_writer.h" #include "index_fst_automation.h" +#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) +typedef struct Fst Fst; typedef struct FstNode FstNode; -#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) +typedef enum { Included, Excluded, Unbounded} FstBound; + +typedef struct FstBoundWithData { + FstSlice data; + FstBound type; +} FstBoundWithData; + +typedef struct FstStreamBuilder { + Fst *fst; + AutomationCtx *aut; + FstBoundWithData *min; + FstBoundWithData *max; +} FstStreamBuilder, FstStreamWithStateBuilder; typedef struct FstRange { uint64_t start; @@ -39,16 +53,9 @@ typedef struct FstRange { typedef enum {GE, GT, LE, LT} RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State; - typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType; -typedef enum { Included, Excluded, Unbounded} FstBound; -typedef struct FstBoundWithData { - FstSlice data; - FstBound type; -} FstBoundWithData; - FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data); bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); @@ -60,8 +67,6 @@ typedef struct FstOutput { Output out; } FstOutput; - - /* * * UnFinished node and helper function @@ -275,6 +280,9 @@ FstNode* fstGetRoot(Fst *fst); FstType fstGetType(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst); Output fstEmptyFinalOutput(Fst *fst, bool *null); +FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); +FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); + bool fstVerify(Fst *fst); @@ -291,13 +299,13 @@ typedef struct StreamState { void streamStateDestroy(void *s); typedef struct StreamWithState { - Fst *fst; - Automation *aut; - SArray *inp; - FstOutput emptyOutput; - SArray *stack; // + Fst *fst; + AutomationCtx *aut; + SArray *inp; + FstOutput emptyOutput; + SArray *stack; // FstBoundWithData *endAt; -} StreamWithState ; +} StreamWithState; typedef struct StreamWithStateResult { FstSlice data; @@ -310,19 +318,13 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta void swsResultDestroy(StreamWithStateResult *result); typedef void* (*StreamCallback)(void *); -StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) ; +StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ; void streamWithStateDestroy(StreamWithState *sws); bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); -StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); -typedef struct FstStreamBuilder { - Fst *fst; - Automation *aut; - FstBoundWithData *min; - FstBoundWithData *max; -} FstStreamBuilder; +StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut); +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); // set up bound range // refator, simple code by marco diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 480d3110c418766d8f99e72bffeed0256451e845..c2ab61bf5f490601c503dde46a49c7c2323dad39 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -19,33 +19,50 @@ extern "C" { #endif +#include "index_fst_util.h" typedef struct AutomationCtx AutomationCtx; +typedef enum AutomationType { + AUTOMATION_PREFIX, + AUTMMATION_MATCH +} AutomationType; + typedef struct StartWith { AutomationCtx *autoSelf; } StartWith; typedef struct Complement { AutomationCtx *autoSelf; - } Complement; // automation typedef struct AutomationCtx { -// automation interface + AutomationType type; void *data; } AutomationCtx; -typedef struct Automation { - void* (*start)() ; - bool (*isMatch)(void *); - bool (*canMatch)(void *data); - bool (*willAlwaysMatch)(void *state); - void* (*accept)(void *state, uint8_t byte); - void* (*acceptEof)(void *state); - void *data; -} Automation; + +typedef enum StartWithStateKind { Done, Running } StartWithStateKind; + +typedef struct StartWithStateValue { + StartWithStateKind kind; + void *value; +} StartWithStateValue; + +typedef struct AutomationFunc { + void* (*start)(AutomationCtx *ctx) ; + bool (*isMatch)(AutomationCtx *ctx, void *); + bool (*canMatch)(AutomationCtx *ctx, void *data); + bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); + void* (*accept)(AutomationCtx *ctx, void *state, uint8_t byte); + void* (*acceptEof)(AutomationCtx *ct, void *state); +} AutomationFunc; + +AutomationCtx *automCtxCreate(void *data, AutomationType type); +void automCtxDestroy(AutomationCtx *ctx); + +extern AutomationFunc automFuncs[]; #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index e7e9be5263e5de40a5d5388a12807e2de2654026..07f1e343bd8014607939db357aab756c4c7b22ef 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -591,14 +591,14 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { uint8_t *data = fstSliceData(&t, &len); int i = 0; for(; i < len; i++) { - //uint8_t v = slice->data[slice->start + i]; - ////slice->data[slice->start + i]; uint8_t v = data[i]; if (v == b) { + fstSliceDestroy(&t); return node->nTrans - i - 1; // bug } } if (i == len) { *null = true; } + fstSliceDestroy(&t); } } @@ -634,7 +634,7 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { } else if (st.state == OneTrans) { FstSlice data = fstSliceCopy(slice, 0, addr); PackSizes sz = fstStateSizes(&st, &data); - n->data = fstSliceCopy(slice, 0, addr); + n->data = data; n->version = version; n->state = st; n->start = addr; @@ -852,8 +852,9 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { FstSlice *input = &bs; if (fstSliceIsEmpty(&b->last)) { + fstSliceDestroy(&b->last); // deep copy or not - b->last = fstSliceCopy(&bs, input->start, input->end); + b->last = fstSliceDeepCopy(&bs, input->start, input->end); } else { int comp = fstSliceCompare(&b->last, &bs); if (comp == 0 && ckDup) { @@ -863,7 +864,7 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { } // deep copy or not fstSliceDestroy(&b->last); - b->last = fstSliceCopy(&bs, input->start, input->end); + b->last = fstSliceDeepCopy(&bs, input->start, input->end); } return Ordered; } @@ -1071,25 +1072,30 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { tOut += trn.out; root = fstGetNode(fst, trn.addr); taosArrayPush(nodes, &root); - //fstNodeDestroy(root); } if (!FST_NODE_IS_FINAL(root)) { return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); } + for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { FstNode **node = (FstNode **)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); + fst->root = NULL; - - *out = tOut; return true; } +FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { + return fstStreamBuilderCreate(fst, ctx); +} +FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { + return fstStreamBuilderCreate(fst, ctx); +} FstNode *fstGetRoot(Fst *fst) { if (fst->root != NULL) { @@ -1176,7 +1182,7 @@ void fstBoundDestroy(FstBoundWithData *bound) { free(bound); } -StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) { +StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) { StreamWithState *sws = calloc(1, sizeof(StreamWithState)); if (sws == NULL) { return NULL; } @@ -1203,6 +1209,8 @@ void streamWithStateDestroy(StreamWithState *sws) { } bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { + + AutomationCtx *aut = sws->aut; if (fstBoundWithDataIsEmpty(min)) { if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); @@ -1210,7 +1218,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { StreamState s = {.node = fstGetRoot(sws->fst), .trans = 0, .out = {.null = false, .out = 0}, - .autState = sws->aut->start()}; // auto.start callback + .autState = automFuncs[aut->type].start(aut)}; // auto.start callback taosArrayPush(sws->stack, &s); return true; } @@ -1228,7 +1236,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { FstNode *node = fstGetRoot(sws->fst); Output out = 0; - void* autState = sws->aut->start(); + //void* autState = sws->aut->start(); + void* autState = automFuncs[aut->type].start(aut); int32_t len; uint8_t *data = fstSliceData(key, &len); @@ -1240,7 +1249,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { FstTransition trn; fstNodeGetTransitionAt(node, res, &trn); void *preState = autState; - autState = sws->aut->accept(preState, b); + // autState = sws->aut->accept(preState, b); + autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); StreamState s = {.node = node, .trans = res + 1, @@ -1297,6 +1307,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { } StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) { + AutomationCtx *aut = sws->aut; FstOutput output = sws->emptyOutput; if (output.null == false) { FstSlice emptySlice = fstSliceCreate(NULL, 0); @@ -1305,15 +1316,15 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); return NULL; } - void* start = sws->aut->start(); - if (sws->aut->isMatch(start)) { + void *start = automFuncs[aut->type].start(aut); + if (automFuncs[aut->type].isMatch(aut, start)) { FstSlice s = fstSliceCreate(NULL, 0); return swsResultCreate(&s, output, callback(start)); } } while (taosArrayGetSize(sws->stack) > 0) { StreamState *p = (StreamState *)taosArrayPop(sws->stack); - if (p->trans >= FST_NODE_LEN(p->node) || !sws->aut->canMatch(p->autState)) { + if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } @@ -1323,16 +1334,18 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb FstTransition trn; fstNodeGetTransitionAt(p->node, p->trans, &trn); Output out = p->out.out + trn.out; - void* nextState = sws->aut->accept(p->autState, trn.inp); + void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); void* tState = callback(nextState); - bool isMatch = sws->aut->isMatch(nextState); + bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); + //bool isMatch = sws->aut->isMatch(nextState); FstNode *nextNode = fstGetNode(sws->fst, trn.addr); taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { - void *eofState = sws->aut->acceptEof(nextState); + //void *eofState = sws->aut->acceptEof(nextState); + void *eofState = automFuncs[aut->type].acceptEof(aut, nextState); if (eofState != NULL) { - isMatch = sws->aut->isMatch(eofState); + isMatch = automFuncs[aut->type].isMatch(aut, eofState); } } StreamState s1 = { .node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; @@ -1390,7 +1403,7 @@ void streamStateDestroy(void *s) { //free(s->autoState); } -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); if (NULL == b) { return NULL; } @@ -1433,3 +1446,5 @@ FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, Rang + + diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 3d5efd30f3c81ff7660b6a3ec1f4c5ec7b147af5..6a08b41b12ec08f534cd087b7546bc68d86d21c9 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -13,3 +13,94 @@ * along with this program. If not, see . */ +#include "index_fst_automation.h" + + +// prefix query, impl later +static void* prefixStart(AutomationCtx *ctx) { + StartWithStateValue *data = (StartWithStateValue *)(ctx->data); + return data; +}; +static bool prefixIsMatch(AutomationCtx *ctx, void *data) { + return true; +} +static bool prefixCanMatch(AutomationCtx *ctx, void *data) { + return true; +} +static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { + return true; +} +static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { + return NULL; +} +static void* prefixAcceptEof(AutomationCtx *ctx, void *state) { + return NULL; +} + +// pattern query, impl later + +static void* patternStart(AutomationCtx *ctx) { + return NULL; +} +static bool patternIsMatch(AutomationCtx *ctx, void *data) { + return true; +} +static bool patternCanMatch(AutomationCtx *ctx, void *data) { + return true; +} +static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { + return true; +} + +static void* patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { + return NULL; +} + +static void* patternAcceptEof(AutomationCtx *ctx, void *state) { + return NULL; +} + +AutomationFunc automFuncs[] = {{ + prefixStart, + prefixIsMatch, + prefixCanMatch, + prefixWillAlwaysMatch, + prefixAccept, + prefixAcceptEof + }, + { + patternStart, + patternIsMatch, + patternCanMatch, + patternWillAlwaysMatch, + patternAccept, + patternAcceptEof + } + // add more search type +}; + +AutomationCtx* automCtxCreate(void *data, AutomationType type) { + AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); + if (ctx == NULL) { return NULL; } + + if (type == AUTOMATION_PREFIX) { + StartWithStateValue *swsv = (StartWithStateValue *)calloc(1, sizeof(StartWithStateValue)); + swsv->kind = Done; + swsv->value = NULL; + ctx->data = (void *)swsv; + } else if (type == AUTMMATION_MATCH) { + + } else { + // add more search type + } + + ctx->type = type; + return ctx; +} +void automCtxDestroy(AutomationCtx *ctx) { + if (ctx->type == AUTOMATION_PREFIX) { + free(ctx->data); + } else if (ctx->type == AUTMMATION_MATCH) { + } + free(ctx); +} diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 928c3875b07e3ac1caa03d0c9fef175c14ab36f8..0cfb0fedc37706bfddebd7c025690baf380be014 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -65,6 +65,7 @@ class FstReadMemory { ~FstReadMemory() { fstCountingWriterDestroy(_w); + fstDestroy(_fst); fstSliceDestroy(&_s); } @@ -129,10 +130,12 @@ class FstReadMemory { //} +#define L 100 +#define M 100 +#define N 100 int Performance_fstWriteRecords(FstWriter *b) { std::string str("aa"); - int L = 100, M = 100, N = 10; for (int i = 0; i < L; i++) { str[0] = 'a' + i; str.resize(2); @@ -150,22 +153,29 @@ int Performance_fstWriteRecords(FstWriter *b) { } void Performance_fstReadRecords(FstReadMemory *m) { - std::string str("a"); - for (int i = 0; i < 50; i++) { - //std::string str("aa"); - str.push_back('a'); - uint64_t out, cost; - bool ok = m->GetWithTimeCostUs(str, &out, &cost); - if (ok == true) { - printf("success to get (%s, %" PRId64"), time cost: %" PRId64")\n", str.c_str(), out, cost); - } else { - printf("failed to get(%s)\n", str.c_str()); - } - } + std::string str("aa"); + for (int i = 0; i < M; i++) { + str[0] = 'a' + i; + str.resize(2); + for(int j = 0; j < N; j++) { + str[1] = 'a' + j; + str.resize(2); + for (int k = 0; k < L; k++) { + str.push_back('a'); + uint64_t val, cost; + if (m->GetWithTimeCostUs(str, &val, &cost)) { + printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost); + } else { + printf("failed to get key: %s\n", str.c_str()); + } + } + } + } } void checkFstPerf() { FstWriter *fw = new FstWriter; int64_t s = taosGetTimestampUs(); + int num = Performance_fstWriteRecords(fw); int64_t e = taosGetTimestampUs(); printf("write %d record cost %" PRId64"us\n", num, e - s); @@ -173,13 +183,11 @@ void checkFstPerf() { FstReadMemory *m = new FstReadMemory(1024 * 64); if (m->init()) { - uint64_t val; - if(m->Get("aaaaaaa", &val)) { - std::cout << "succes to Get val: " << val << std::endl; - } else { - std::cout << "failed to Get " << std::endl; - } + printf("success to init fst read"); } + Performance_fstReadRecords(m); + + delete m; } diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index cb3d459de6b707089e42f73e37ea4797fc991fbb..954bc29e62e15559663c8bdc61a6e7107d824a13 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -121,7 +121,7 @@ typedef struct SRelationInfo { typedef struct SCreatedTableInfo { SToken name; // table name token - SToken stableName; // super table name token , for using clause + SToken stbName; // super table name token , for using clause SArray *pTagNames; // create by using super table, tag name SArray *pTagVals; // create by using super table, tag value char *fullname; // table full name diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h new file mode 100644 index 0000000000000000000000000000000000000000..7d2e6e3aec336c2e1158501c7197ebdf790fa79b --- /dev/null +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DATABLOCKMGT_H +#define TDENGINE_DATABLOCKMGT_H + +#include "catalog.h" +#include "os.h" +#include "ttypes.h" +#include "tname.h" + +#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) + +typedef enum EOrderStatus { + ORDER_STATUS_UNKNOWN = 0, + ORDER_STATUS_ORDERED = 1, + ORDER_STATUS_DISORDERED = 2, +} EOrderStatus; + +typedef enum EValStat { + VAL_STAT_HAS = 0x0, // 0 means has val + VAL_STAT_NONE = 0x01, // 1 means no val +} EValStat; + +typedef enum ERowCompareStat { + ROW_COMPARE_NO_NEED = 0, + ROW_COMPARE_NEED = 1, +} ERowCompareStat; + +typedef struct SBoundColumn { + int32_t offset; // all column offset value + int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future + uint8_t valStat; // EValStat. denote if current column bound or not(0 means has val, 1 means no val) +} SBoundColumn; + +typedef struct { + uint16_t schemaColIdx; + uint16_t boundIdx; + uint16_t finalIdx; +} SBoundIdxInfo; + +typedef struct SParsedDataColInfo { + int16_t numOfCols; + int16_t numOfBound; + uint16_t flen; // TODO: get from STSchema + uint16_t allNullLen; // TODO: get from STSchema + uint16_t extendedVarLen; + int32_t *boundedColumns; // bound column idx according to schema + SBoundColumn *cols; + SBoundIdxInfo *colIdxInfo; + int8_t orderStatus; // bound columns +} SParsedDataColInfo; + +typedef struct SMemRowInfo { + int32_t dataLen; // len of SDataRow + int32_t kvLen; // len of SKVRow +} SMemRowInfo; + +typedef struct { + uint8_t memRowType; // default is 0, that is SDataRow + uint8_t compareStat; // 0 no need, 1 need compare + TDRowTLenT kvRowInitLen; + SMemRowInfo *rowInfo; +} SMemRowBuilder; + +typedef struct STableDataBlocks { + int8_t tsSource; // where does the UNIX timestamp come from, server or client + bool ordered; // if current rows are ordered or not + int64_t vgId; // virtual group id + int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending + int32_t numOfTables; // number of tables in current submit block + int32_t rowSize; // row size for current table + uint32_t nAllocSize; + uint32_t headerSize; // header for table info (uid, tid, submit metadata) + uint32_t size; + STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache + char *pData; + bool cloned; + STagData tagData; + + SParsedDataColInfo boundColumnInfo; + SMemRowBuilder rowBuilder; +} STableDataBlocks; + +static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) { + memRowSetType(row, memRowType); + if (isDataRowT(memRowType)) { + dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion); + dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen)); + } else { + ASSERT(nBoundCols > 0); + memRowSetKvVersion(row, pBlock->pTableMeta->sversion); + kvRowSetNCols(memRowKvBody(row), nBoundCols); + kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); + } +} + +static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { + ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize); + return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen; +} + +// Applicable to consume by one row +static FORCE_INLINE void appendMemRowColValEx(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, + int8_t colType, int32_t toffset, int32_t *dataLen, int32_t *kvLen, + uint8_t compareStat) { + tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset); + if (compareStat == ROW_COMPARE_NEED) { + tdGetColAppendDeltaLen(value, colType, dataLen, kvLen); + } +} + +static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd, + int32_t idx, int32_t *toffset) { + int32_t schemaIdx = 0; + if (IS_DATA_COL_ORDERED(spd)) { + schemaIdx = spd->boundedColumns[idx]; + if (isDataRowT(memRowType)) { + *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart + } else { + *toffset = idx * sizeof(SColIdx); // the offset of SColIdx + } + } else { + ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); + schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; + if (isDataRowT(memRowType)) { + *toffset = (spd->cols + schemaIdx)->toffset; + } else { + *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); + } + } +} + +static FORCE_INLINE void convertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) { + if (isDataRow(row)) { + if (kvLen < (dataLen * KVRatioConvert)) { + memRowSetConvert(row); + } + } else if (kvLen > dataLen) { + memRowSetConvert(row); + } +} + +static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { + pBlocks->tid = pTableMeta->suid; + pBlocks->uid = pTableMeta->uid; + pBlocks->sversion = pTableMeta->sversion; + + if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } else { + pBlocks->numOfRows += numOfRows; + return TSDB_CODE_SUCCESS; + } +} + +int32_t schemaIdxCompar(const void *lhs, const void *rhs); +int32_t boundIdxCompar(const void *lhs, const void *rhs); +void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); +void destroyBoundColumnInfo(SParsedDataColInfo* pColList); +int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); +int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); +int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap); + +#endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/insertParser.h b/source/libs/parser/inc/insertParser.h index 49e678cd541eae0e6b1af9473d7405928089fbfe..b0191b155d7b6969fdfd41942b612b603d65fc52 100644 --- a/source/libs/parser/inc/insertParser.h +++ b/source/libs/parser/inc/insertParser.h @@ -16,4 +16,16 @@ #ifndef TDENGINE_INSERTPARSER_H #define TDENGINE_INSERTPARSER_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "parser.h" + +int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo); + +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_INSERTPARSER_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index e040c2187920c11f5f8910e61c315d3d8451e508..e3925d34466562eca7b1bc1dd18f59aaaf02c401 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -26,14 +26,6 @@ extern "C" { struct SSqlNode; -typedef struct SInsertStmtInfo { - SHashObj *pTableBlockHashList; // data block for each table - SArray *pDataBlocks; // SArray. Merged submit block for each vgroup - int8_t schemaAttached; // denote if submit block is built with table schema or not - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - uint32_t insertType; // insert data from [file|sql statement| bound statement] - char *sql; // current sql statement position -} SInsertStmtInfo; typedef struct SInternalField { TAOS_FIELD field; diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 12ffe696c13ebb88b2bfd7faf1b388a1c9295ab6..6c95c4327b99aa08e570ce697e8e3504271704f5 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -46,7 +46,7 @@ SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index); int32_t parserValidateIdToken(SToken* pToken); int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); -int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo, const char* sourceStr); +int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo); @@ -61,6 +61,8 @@ void cleanupColumnCond(SArray** pCond); uint32_t convertRelationalOperator(SToken *pToken); int32_t getExprFunctionId(SExprInfo *pExprInfo); +STableMeta* tableMetaDup(const STableMeta* pTableMeta); + #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index ca281c9f1cec557a18fc37027dbd507ac583092b..da975b966ea2360a9c9672f15669c8f1bf5f4ff9 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -112,13 +112,13 @@ cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). { } cmd ::= SHOW dbPrefix(X) STABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &X, 0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &X, 0); } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { SToken token; tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &Y); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { diff --git a/source/libs/parser/inc/ttoken.h b/source/libs/parser/inc/ttoken.h index bacabe299e8c7c6dff957e9dd9addc10e2b2aef7..c041edf2b25d2ea9e270e3370e2df68ebcd19b8f 100644 --- a/source/libs/parser/inc/ttoken.h +++ b/source/libs/parser/inc/ttoken.h @@ -44,7 +44,7 @@ typedef struct SToken { * @param tokenType * @return */ -uint32_t tGetToken(char *z, uint32_t *tokenType); +uint32_t tGetToken(const char *z, uint32_t *tokenType); /** * enhanced tokenizer for sql string. @@ -54,7 +54,7 @@ uint32_t tGetToken(char *z, uint32_t *tokenType); * @param isPrevOptr * @return */ -SToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr); +SToken tStrGetToken(const char *str, int32_t *i, bool isPrevOptr); /** * check if it is a keyword or not diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index 3b7d1cbc2941e5a99468d483714cdd27351c8069..d12278632ba815090e604c67b3b68df5a8c4f4ab 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -191,7 +191,7 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) { pExpr->type = SQL_NODE_EXPR; if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { - char* endPos = pRight->exprToken.z + pRight->exprToken.n; + const char* endPos = pRight->exprToken.z + pRight->exprToken.n; pExpr->exprToken.z = pLeft->exprToken.z; pExpr->exprToken.n = (uint32_t)(endPos - pExpr->exprToken.z); pExpr->exprToken.type = pLeft->exprToken.type; @@ -634,7 +634,7 @@ SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, info.name = *pToken; info.pTagNames = pTagNames; info.pTagVals = pTagVals; - info.stableName = *pTableName; + info.stbName = *pTableName; info.igExist = (igExists->n > 0)? 1:0; return info; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index ce47e50c1e48304ebe3ea5be1001e2cf95ecc63e..d2813886b323c41658f07b50d46da7891d0f66d2 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4078,7 +4078,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer #endif SCatalogReq req = {0}; - SCatalogRsp data = {0}; + SMetaData data = {0}; // TODO: check if the qnode info has been cached already req.qNodeRequired = true; diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c new file mode 100644 index 0000000000000000000000000000000000000000..3138f0c1d8c8fe0bdca6b835e1c0afe9bb0fe407 --- /dev/null +++ b/source/libs/parser/src/dataBlockMgt.c @@ -0,0 +1,652 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "dataBlockMgt.h" + +#include "catalog.h" +#include "parserUtil.h" +#include "queryInfoUtil.h" +#include "taosmsg.h" + +#define IS_RAW_PAYLOAD(t) \ + (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + +typedef struct SBlockKeyTuple { + TSKEY skey; + void* payloadAddr; +} SBlockKeyTuple; + +typedef struct SBlockKeyInfo { + int32_t maxBytesAlloc; + SBlockKeyTuple* pKeyTuple; +} SBlockKeyInfo; + +static int32_t rowDataCompar(const void *lhs, const void *rhs) { + TSKEY left = *(TSKEY *)lhs; + TSKEY right = *(TSKEY *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} + +void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols) { + pColList->numOfCols = numOfCols; + pColList->numOfBound = numOfCols; + pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode + pColList->boundedColumns = calloc(pColList->numOfCols, sizeof(int32_t)); + pColList->cols = calloc(pColList->numOfCols, sizeof(SBoundColumn)); + pColList->colIdxInfo = NULL; + pColList->flen = 0; + pColList->allNullLen = 0; + + int32_t nVar = 0; + for (int32_t i = 0; i < pColList->numOfCols; ++i) { + uint8_t type = pSchema[i].type; + if (i > 0) { + pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; + pColList->cols[i].toffset = pColList->flen; + } + pColList->flen += TYPE_BYTES[type]; + switch (type) { + case TSDB_DATA_TYPE_BINARY: + pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); + ++nVar; + break; + case TSDB_DATA_TYPE_NCHAR: + pColList->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + ++nVar; + break; + default: + break; + } + pColList->boundedColumns[i] = pSchema[i].colId; + } + pColList->allNullLen += pColList->flen; + pColList->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); +} + +int32_t schemaIdxCompar(const void *lhs, const void *rhs) { + uint16_t left = *(uint16_t *)lhs; + uint16_t right = *(uint16_t *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} + +int32_t boundIdxCompar(const void *lhs, const void *rhs) { + uint16_t left = *(uint16_t *)POINTER_SHIFT(lhs, sizeof(uint16_t)); + uint16_t right = *(uint16_t *)POINTER_SHIFT(rhs, sizeof(uint16_t)); + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} + +void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { + tfree(pColList->boundedColumns); + tfree(pColList->cols); + tfree(pColList->colIdxInfo); +} + +static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { + STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); + if (dataBuf == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + dataBuf->nAllocSize = (uint32_t)defaultSize; + dataBuf->headerSize = startOffset; + + // the header size will always be the startOffset value, reserved for the subumit block header + if (dataBuf->nAllocSize <= dataBuf->headerSize) { + dataBuf->nAllocSize = dataBuf->headerSize * 2; + } + + //dataBuf->pData = calloc(1, dataBuf->nAllocSize); + dataBuf->pData = malloc(dataBuf->nAllocSize); + if (dataBuf->pData == NULL) { + tfree(dataBuf); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); + + //Here we keep the tableMeta to avoid it to be remove by other threads. + dataBuf->pTableMeta = tableMetaDup(pTableMeta); + + SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; + SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); + setBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); + + dataBuf->ordered = true; + dataBuf->prevTS = INT64_MIN; + dataBuf->rowSize = rowSize; + dataBuf->size = startOffset; + dataBuf->tsSource = -1; + dataBuf->vgId = dataBuf->pTableMeta->vgId; + + // tNameAssign(&dataBuf->tableName, name); + + assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); + + *dataBlocks = dataBuf; + return TSDB_CODE_SUCCESS; +} + +int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { + *dataBlocks = NULL; + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); + if (t1 != NULL) { + *dataBlocks = *t1; + } + + if (*dataBlocks == NULL) { + int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, pTableMeta, dataBlocks); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); + if (pBlockList) { + taosArrayPush(pBlockList, dataBlocks); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t getRowExpandSize(STableMeta* pTableMeta) { + int32_t result = TD_MEM_ROW_DATA_HEAD_SIZE; + int32_t columns = getNumOfColumns(pTableMeta); + SSchema* pSchema = getTableColumnSchema(pTableMeta); + for (int32_t i = 0; i < columns; i++) { + if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { + result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; + } + } + return result; +} + +/** + * TODO: Move to tdataformat.h and refactor when STSchema available. + * - fetch flen and toffset from STSChema and remove param spd + */ +static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, SParsedDataColInfo *spd) { + ASSERT(isKvRow(src)); + SKVRow kvRow = memRowKvBody(src); + SDataRow dataRow = memRowDataBody(dest); + + memRowSetType(dest, SMEM_ROW_DATA); + dataRowSetVersion(dataRow, memRowKvVersion(src)); + dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen)); + + int32_t kvIdx = 0; + for (int i = 0; i < nCols; ++i) { + SSchema *schema = pSchema + i; + void * val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx); + tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type, + (spd->cols + i)->toffset); + } +} + +// TODO: Move to tdataformat.h and refactor when STSchema available. +static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, SParsedDataColInfo *spd) { + ASSERT(isDataRow(src)); + + SDataRow dataRow = memRowDataBody(src); + SKVRow kvRow = memRowKvBody(dest); + + memRowSetType(dest, SMEM_ROW_KV); + memRowSetKvVersion(kvRow, dataRowVersion(dataRow)); + kvRowSetNCols(kvRow, nBoundCols); + kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); + + int32_t toffset = 0, kvOffset = 0; + for (int i = 0; i < nCols; ++i) { + if ((spd->cols + i)->valStat == VAL_STAT_HAS) { + SSchema *schema = pSchema + i; + toffset = (spd->cols + i)->toffset; + void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE); + tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset); + kvOffset += sizeof(SColIdx); + } + } +} + +// TODO: Move to tdataformat.h and refactor when STSchema available. +static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) { + STableMeta * pTableMeta = pBlock->pTableMeta; + STableComInfo tinfo = getTableInfo(pTableMeta); + SSchema * pSchema = getTableColumnSchema(pTableMeta); + SParsedDataColInfo *spd = &pBlock->boundColumnInfo; + + ASSERT(dest != src); + + if (isDataRow(src)) { + // TODO: Can we use pBlock -> numOfParam directly? + ASSERT(spd->numOfBound > 0); + convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd); + } else { + convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd); + } +} + +void destroyDataBlock(STableDataBlocks* pDataBlock) { + if (pDataBlock == NULL) { + return; + } + + tfree(pDataBlock->pData); + if (!pDataBlock->cloned) { + // free the refcount for metermeta + if (pDataBlock->pTableMeta != NULL) { + tfree(pDataBlock->pTableMeta); + } + + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + } + tfree(pDataBlock); +} + +void* destroyBlockArrayList(SArray* pDataBlockList) { + if (pDataBlockList == NULL) { + return NULL; + } + + size_t size = taosArrayGetSize(pDataBlockList); + for (int32_t i = 0; i < size; i++) { + void* d = taosArrayGetP(pDataBlockList, i); + destroyDataBlock(d); + } + + taosArrayDestroy(pDataBlockList); + return NULL; +} + +// data block is disordered, sort it in ascending order +void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { + SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; + + // size is less than the total size, since duplicated rows may be removed yet. + assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size); + + if (!dataBuf->ordered) { + char *pBlockData = pBlocks->data; + qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); + + int32_t i = 0; + int32_t j = 1; + + while (j < pBlocks->numOfRows) { + TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); + TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); + + if (ti == tj) { + ++j; + continue; + } + + int32_t nextPos = (++i); + if (nextPos != j) { + memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); + } + + ++j; + } + + dataBuf->ordered = true; + + pBlocks->numOfRows = i + 1; + dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; + } + + dataBuf->prevTS = INT64_MIN; +} + +// data block is disordered, sort it in ascending order +int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) { + SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; + int16_t nRows = pBlocks->numOfRows; + + // size is less than the total size, since duplicated rows may be removed yet. + + // allocate memory + size_t nAlloc = nRows * sizeof(SBlockKeyTuple); + if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { + char *tmp = realloc(pBlkKeyInfo->pKeyTuple, nAlloc); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; + pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; + } + memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); + + int32_t extendedRowSize = getExtendedRowSize(dataBuf); + SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + char * pBlockData = pBlocks->data; + int n = 0; + while (n < nRows) { + pBlkKeyTuple->skey = memRowKey(pBlockData); + pBlkKeyTuple->payloadAddr = pBlockData; + + // next loop + pBlockData += extendedRowSize; + ++pBlkKeyTuple; + ++n; + } + + if (!dataBuf->ordered) { + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); + + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + int32_t i = 0; + int32_t j = 1; + while (j < nRows) { + TSKEY ti = (pBlkKeyTuple + i)->skey; + TSKEY tj = (pBlkKeyTuple + j)->skey; + + if (ti == tj) { + ++j; + continue; + } + + int32_t nextPos = (++i); + if (nextPos != j) { + memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); + } + ++j; + } + + dataBuf->ordered = true; + pBlocks->numOfRows = i + 1; + } + + dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; + dataBuf->prevTS = INT64_MIN; + + return 0; +} + +// Erase the empty space reserved for binary data +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, int8_t schemaAttached, bool isRawPayload) { + // TODO: optimize this function, handle the case while binary is not presented + STableMeta* pTableMeta = pTableDataBlock->pTableMeta; + STableComInfo tinfo = getTableInfo(pTableMeta); + SSchema* pSchema = getTableColumnSchema(pTableMeta); + + SSubmitBlk* pBlock = pDataBlock; + memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); + pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk); + + int32_t flen = 0; // original total length of row + + // schema needs to be included into the submit data block + if (schemaAttached) { + int32_t numOfCols = getNumOfColumns(pTableDataBlock->pTableMeta); + for(int32_t j = 0; j < numOfCols; ++j) { + STColumn* pCol = (STColumn*) pDataBlock; + pCol->colId = htons(pSchema[j].colId); + pCol->type = pSchema[j].type; + pCol->bytes = htons(pSchema[j].bytes); + pCol->offset = 0; + + pDataBlock = (char*)pDataBlock + sizeof(STColumn); + flen += TYPE_BYTES[pSchema[j].type]; + } + + int32_t schemaSize = sizeof(STColumn) * numOfCols; + pBlock->schemaLen = schemaSize; + } else { + if (isRawPayload) { + for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { + flen += TYPE_BYTES[pSchema[j].type]; + } + } + pBlock->schemaLen = 0; + } + + char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); + pBlock->dataLen = 0; + int32_t numOfRows = htons(pBlock->numOfRows); + + if (isRawPayload) { + for (int32_t i = 0; i < numOfRows; ++i) { + SMemRow memRow = (SMemRow)pDataBlock; + memRowSetType(memRow, SMEM_ROW_DATA); + SDataRow trow = memRowDataBody(memRow); + dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen)); + dataRowSetVersion(trow, pTableMeta->sversion); + + int toffset = 0; + for (int32_t j = 0; j < tinfo.numOfColumns; j++) { + tdAppendColVal(trow, p, pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + p += pSchema[j].bytes; + } + + pDataBlock = (char*)pDataBlock + memRowTLen(memRow); + pBlock->dataLen += memRowTLen(memRow); + } + } else { + for (int32_t i = 0; i < numOfRows; ++i) { + char* payload = (blkKeyTuple + i)->payloadAddr; + if (isNeedConvertRow(payload)) { + convertSMemRow(pDataBlock, payload, pTableDataBlock); + TDRowTLenT rowTLen = memRowTLen(pDataBlock); + pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); + pBlock->dataLen += rowTLen; + } else { + TDRowTLenT rowTLen = memRowTLen(payload); + memcpy(pDataBlock, payload, rowTLen); + pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); + pBlock->dataLen += rowTLen; + } + } + } + + int32_t len = pBlock->dataLen + pBlock->schemaLen; + pBlock->dataLen = htonl(pBlock->dataLen); + pBlock->schemaLen = htonl(pBlock->schemaLen); + + return len; +} + +static void extractTableNameList(SHashObj* pHashObj, bool freeBlockMap) { + // todo +} + +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap) { + const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); + int code = 0; + bool isRawPayload = IS_RAW_PAYLOAD(payloadType); + SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); + + STableDataBlocks** p = taosHashIterate(pHashObj, NULL); + STableDataBlocks* pOneTableBlock = *p; + SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock + while (pOneTableBlock) { + SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; + if (pBlocks->numOfRows > 0) { + STableDataBlocks* dataBuf = NULL; + int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, + INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + if (ret != TSDB_CODE_SUCCESS) { + taosHashCleanup(pVnodeDataBlockHashList); + destroyBlockArrayList(pVnodeDataBlockList); + tfree(blkKeyInfo.pKeyTuple); + return ret; + } + + // the maximum expanded size in byte when a row-wise data is converted to SDataRow format + int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; + int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); + + if (dataBuf->nAllocSize < destSize) { + dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); + char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); + if (tmp != NULL) { + dataBuf->pData = tmp; + } else { // failed to allocate memory, free already allocated memory and return error code + taosHashCleanup(pVnodeDataBlockHashList); + destroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + if (isRawPayload) { + sortRemoveDataBlockDupRowsRaw(pOneTableBlock); + } else { + if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { + taosHashCleanup(pVnodeDataBlockHashList); + destroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); + return code; + } + ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); + } + + int32_t len = pBlocks->numOfRows * + (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); + + pBlocks->tid = htonl(pBlocks->tid); + pBlocks->uid = htobe64(pBlocks->uid); + pBlocks->sversion = htonl(pBlocks->sversion); + pBlocks->numOfRows = htons(pBlocks->numOfRows); + pBlocks->schemaLen = 0; + + // erase the empty space reserved for binary data + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload); + assert(finalLen <= len); + + dataBuf->size += (finalLen + sizeof(SSubmitBlk)); + assert(dataBuf->size <= dataBuf->nAllocSize); + + // the length does not include the SSubmitBlk structure + pBlocks->dataLen = htonl(finalLen); + dataBuf->numOfTables += 1; + + pBlocks->numOfRows = 0; + } + + p = taosHashIterate(pHashObj, p); + if (p == NULL) { + break; + } + + pOneTableBlock = *p; + } + + extractTableNameList(pHashObj, freeBlockMap); + + // free the table data blocks; + taosHashCleanup(pVnodeDataBlockHashList); + tfree(blkKeyInfo.pKeyTuple); + + return TSDB_CODE_SUCCESS; +} + +int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) { + size_t remain = pDataBlock->nAllocSize - pDataBlock->size; + const int factor = 5; + uint32_t nAllocSizeOld = pDataBlock->nAllocSize; + + // expand the allocated size + if (remain < rowSize * factor) { + while (remain < rowSize * factor) { + pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); + remain = pDataBlock->nAllocSize - pDataBlock->size; + } + + char *tmp = realloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); + if (tmp != NULL) { + pDataBlock->pData = tmp; + memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); + } else { + // do nothing, if allocate more memory failed + pDataBlock->nAllocSize = nAllocSizeOld; + *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; + return TSDB_CODE_SUCCESS; +} + +int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen) { + ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols)); + if (nRows > 0) { + // already init(bind multiple rows by single column) + if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) { + return TSDB_CODE_SUCCESS; + } + } + + // default compareStat is ROW_COMPARE_NO_NEED + if (nBoundCols == 0) { // file input + pBuilder->memRowType = SMEM_ROW_DATA; + return TSDB_CODE_SUCCESS; + } else { + float boundRatio = ((float)nBoundCols / (float)nCols); + + if (boundRatio < KVRatioKV) { + pBuilder->memRowType = SMEM_ROW_KV; + return TSDB_CODE_SUCCESS; + } else if (boundRatio > KVRatioData) { + pBuilder->memRowType = SMEM_ROW_DATA; + return TSDB_CODE_SUCCESS; + } + pBuilder->compareStat = ROW_COMPARE_NEED; + + if (boundRatio < KVRatioPredict) { + pBuilder->memRowType = SMEM_ROW_KV; + } else { + pBuilder->memRowType = SMEM_ROW_DATA; + } + } + + pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx); + + if (nRows > 0) { + pBuilder->rowInfo = calloc(nRows, sizeof(SMemRowInfo)); + if (pBuilder->rowInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int i = 0; i < nRows; ++i) { + (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen; + (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen; + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c new file mode 100644 index 0000000000000000000000000000000000000000..fa59bc6ca70915aacded5814ec71778c52170871 --- /dev/null +++ b/source/libs/parser/src/insertParser.c @@ -0,0 +1,881 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "insertParser.h" + +#include "dataBlockMgt.h" +#include "parserInt.h" +#include "parserUtil.h" +#include "queryInfoUtil.h" +#include "tglobal.h" +#include "ttime.h" +#include "ttoken.h" +#include "ttypes.h" + +#define NEXT_TOKEN(pSql, sToken) \ + do { \ + int32_t index = 0; \ + sToken = tStrGetToken(pSql, &index, false); \ + pSql += index; \ + } while (0) + +#define CHECK_CODE(expr) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + terrno = code; \ + return terrno; \ + } \ + } while (0) + +#define CHECK_CODE_1(expr, destroy) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + (void)destroy; \ + terrno = code; \ + return terrno; \ + } \ + } while (0) + +#define CHECK_CODE_2(expr, destroy1, destroy2) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + (void)destroy1; \ + (void)destroy2; \ + terrno = code; \ + return terrno; \ + } \ + } while (0) + +enum { + TSDB_USE_SERVER_TS = 0, + TSDB_USE_CLI_TS = 1, +}; + +typedef struct SInsertParseContext { + SParseContext* pComCxt; + const char* pSql; + SMsgBuf msg; + struct SCatalog* pCatalog; + SMetaData meta; // need release + const STableMeta* pTableMeta; + SHashObj* pTableBlockHashObj; // data block for each table. need release + int32_t totalNum; + SInsertStmtInfo* pOutput; +} SInsertParseContext; + +static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; +static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; + +static bool isNullStr(SToken *pToken) { + return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); +} + +static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) { + errno = 0; + *value = strtold(pToken->z, endPtr); + + // not a valid integer number, return error + if ((*endPtr - pToken->z) != pToken->n) { + return TK_ILLEGAL; + } + + return pToken->type; +} + +static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) { + errno = 0; + int32_t ret = 0; + + char* endPtr = NULL; + if (type == TK_FLOAT) { + double v = strtod(z, &endPtr); + if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) { + ret = -1; + } else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) { + ret = -1; + } else { + *value = (int64_t) round(v); + } + + errno = 0; + return ret; + } + + int32_t radix = 10; + if (type == TK_HEX) { + radix = 16; + } else if (type == TK_BIN) { + radix = 2; + } + + // the string may be overflow according to errno + if (!issigned) { + const char *p = z; + while(*p != 0 && *p == ' ') p++; + if (*p != 0 && *p == '-') { return -1;} + + *value = strtoull(z, &endPtr, radix); + } else { + *value = strtoll(z, &endPtr, radix); + } + + // not a valid integer number, return error + if (endPtr - z != n || errno == ERANGE) { + ret = -1; + } + + errno = 0; + return ret; +} + +static int32_t createInsertStmtInfo(SInsertStmtInfo **pInsertInfo) { + *pInsertInfo = calloc(1, sizeof(SQueryStmtInfo)); + if (NULL == *pInsertInfo) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t skipInsertInto(SInsertParseContext* pCxt) { + SToken sToken; + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_INSERT != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, "keyword INSERT is expected", sToken.z); + } + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_INTO != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, "keyword INTO is expected", sToken.z); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) { + if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z); + } + + SName name = {0}; + strcpy(name.dbname, pCxt->pComCxt->pDbname); + strncpy(name.tname, pStname->z, pStname->n); + taosArrayPush(tableNameList, &name); + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SCatalogReq* pMetaReq) { + pMetaReq->pTableName = taosArrayInit(4, sizeof(SName)); + return buildTableName(pCxt, pStname, pMetaReq->pTableName); +} + +static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { + SCatalogReq req; + CHECK_CODE(buildMetaReq(pCxt, pTname, &req)); + CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, &pCxt->meta)); //TODO + pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0); + return TSDB_CODE_SUCCESS; +} + +// todo speedup by using hash list +static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { + while (start < end) { + if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { + return start; + } + ++start; + } + return -1; +} + +static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { + // once the data block is disordered, we do NOT keep previous timestamp any more + if (!pDataBlocks->ordered) { + return TSDB_CODE_SUCCESS; + } + + TSKEY k = *(TSKEY *)start; + + if (k == INT64_MIN) { + if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { + return -1; + } else if (pDataBlocks->tsSource == -1) { + pDataBlocks->tsSource = TSDB_USE_SERVER_TS; + } + } else { + if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { + return -1; // client time/server time can not be mixed + } else if (pDataBlocks->tsSource == -1) { + pDataBlocks->tsSource = TSDB_USE_CLI_TS; + } + } + + if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { + pDataBlocks->ordered = false; + } + + pDataBlocks->prevTS = k; + return TSDB_CODE_SUCCESS; +} + +static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec, int64_t *time) { + int32_t index = 0; + SToken sToken; + int64_t interval; + int64_t useconds = 0; + const char* pTokenEnd = pCxt->pSql; + + if (pToken->type == TK_NOW) { + useconds = taosGetTimestamp(timePrec); + } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) { + // do nothing + } else if (pToken->type == TK_INTEGER) { + useconds = taosStr2int64(pToken->z); + } else { + // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm); + if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp format", pToken->z); + } + + return TSDB_CODE_SUCCESS; + } + + for (int k = pToken->n; pToken->z[k] != '\0'; k++) { + if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; + if (pToken->z[k] == ',') { + pCxt->pSql = pTokenEnd; + *time = useconds; + return 0; + } + + break; + } + + /* + * time expression: + * e.g., now+12a, now-5h + */ + SToken valueToken; + index = 0; + sToken = tStrGetToken(pTokenEnd, &index, false); + pTokenEnd += index; + + if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) { + index = 0; + valueToken = tStrGetToken(pTokenEnd, &index, false); + pTokenEnd += index; + + if (valueToken.n < 2) { + return buildSyntaxErrMsg(&pCxt->msg, "value expected in timestamp", sToken.z); + } + + char unit = 0; + if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if (sToken.type == TK_PLUS) { + useconds += interval; + } else { + useconds = useconds - interval; + } + + pCxt->pSql = pTokenEnd; + } + + *time = useconds; + return TSDB_CODE_SUCCESS; +} + +typedef int32_t (*FRowAppend)(const void *value, int32_t len, void *param); + +typedef struct SKvParam { + char buf[TSDB_MAX_TAGS_LEN]; + SKVRowBuilder* builder; + SSchema* schema; +} SKvParam; + +static FORCE_INLINE int32_t KvRowAppend(const void *value, int32_t len, void *param) { + SKvParam* pa = (SKvParam*)param; + if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { + STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); + tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, pa->buf); + } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + if (!taosMbsToUcs4(value, len, varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } + varDataSetLen(pa->buf, output); + tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, pa->buf); + } else { + tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, value); + } + return TSDB_CODE_SUCCESS; +} + +typedef struct SMemParam { + SMemRow row; + SSchema* schema; + int32_t toffset; + uint8_t compareStat; + int32_t dataLen; + int32_t kvLen; +} SMemParam; + +static FORCE_INLINE int32_t MemRowAppend(const void *value, int32_t len, void *param) { + SMemParam* pa = (SMemParam*)param; + if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { + char *rowEnd = memRowEnd(pa->row); + STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); + appendMemRowColValEx(pa->row, rowEnd, true, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat); + } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + char * rowEnd = memRowEnd(pa->row); + if (!taosMbsToUcs4(value, len, (char *)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } + varDataSetLen(rowEnd, output); + appendMemRowColValEx(pa->row, rowEnd, false, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat); + } else { + appendMemRowColValEx(pa->row, value, true, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat); + } + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t checkAndTrimValue(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, char* tmpTokenBuf) { + int16_t type = pToken->type; + if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && + type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || + (pToken->n == 0) || (type == TK_RP)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pToken->z); + } + + if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); + } + + // Remove quotation marks + if (TK_STRING == type) { + if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { + return buildSyntaxErrMsg(&pCxt->msg, "too long string", pToken->z); + } + // delete escape character: \\, \', \" + char delim = pToken->z[0]; + int32_t cnt = 0; + int32_t j = 0; + for (uint32_t k = 1; k < pToken->n - 1; ++k) { + if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) { + tmpTokenBuf[j] = pToken->z[k + 1]; + cnt++; + j++; + k++; + continue; + } + tmpTokenBuf[j] = pToken->z[k]; + j++; + } + tmpTokenBuf[j] = 0; + pToken->z = tmpTokenBuf; + pToken->n -= 2 + cnt; + } + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t parseOneValue(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, FRowAppend func, void* param) { + int64_t iv; + int32_t ret; + char * endptr = NULL; + + CHECK_CODE(checkAndTrimValue(pCxt, pToken, pSchema, tmpTokenBuf)); + + if (isNullStr(pToken)) { + if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + int64_t tmpVal = 0; + return func(&tmpVal, pSchema->bytes, param); + } + return func(getNullValue(pSchema->type), 0, param); + } + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + return func(&TRUE_VALUE, pSchema->bytes, param); + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + return func(&FALSE_VALUE, pSchema->bytes, param); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_INTEGER) { + return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + } else if (pToken->type == TK_FLOAT) { + return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); + } + break; + } + case TSDB_DATA_TYPE_TINYINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "data overflow", pToken->z); + } + uint8_t tmpVal = (uint8_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_UTINYINT:{ + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); + } + uint8_t tmpVal = (uint8_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_SMALLINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); + } + int16_t tmpVal = (int16_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_USMALLINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); + } + uint16_t tmpVal = (uint16_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_INT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); + } + int32_t tmpVal = (int32_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_UINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); + } + uint32_t tmpVal = (uint32_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_BIGINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "bigint data overflow", pToken->z); + } + return func(&iv, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_UBIGINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned bigint data overflow", pToken->z); + } + uint64_t tmpVal = (uint64_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_FLOAT: { + double dv; + if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); + } + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); + } + float tmpVal = (float)dv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_DOUBLE: { + double dv; + if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + } + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + } + return func(&dv, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_BINARY: { + // too long values will return invalid sql, not be truncated automatically + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); + } + return func(pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_NCHAR: { + return func(pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_TIMESTAMP: { + int64_t tmpVal; + if (parseTime(pCxt, pToken, timePrec, &tmpVal) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); + } + return func(&tmpVal, pSchema->bytes, param); + } + } + + return TSDB_CODE_FAILED; +} + +// pSql -> tag1_name, ...) +static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { + int32_t nCols = pColList->numOfCols; + + pColList->numOfBound = 0; + memset(pColList->boundedColumns, 0, sizeof(int32_t) * nCols); + for (int32_t i = 0; i < nCols; ++i) { + pColList->cols[i].valStat = VAL_STAT_NONE; + } + + SToken sToken; + bool isOrdered = true; + int32_t lastColIdx = -1; // last column found + while (1) { + NEXT_TOKEN(pCxt->pSql, sToken); + + if (TK_RP == sToken.type) { + break; + } + + int32_t t = lastColIdx + 1; + int32_t index = findCol(&sToken, t, nCols, pSchema); + if (index < 0 && t > 0) { + index = findCol(&sToken, 0, t, pSchema); + isOrdered = false; + } + if (index < 0) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid column/tag name", sToken.z); + } + if (pColList->cols[index].valStat == VAL_STAT_HAS) { + return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", sToken.z); + } + lastColIdx = index; + pColList->cols[index].valStat = VAL_STAT_HAS; + pColList->boundedColumns[pColList->numOfBound] = index; + ++pColList->numOfBound; + } + + pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (!isOrdered) { + pColList->colIdxInfo = calloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + if (NULL == pColList->colIdxInfo) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SBoundIdxInfo* pColIdx = pColList->colIdxInfo; + for (uint16_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = (uint16_t)pColList->boundedColumns[i]; + pColIdx[i].boundIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (uint16_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + memset(&pColList->boundedColumns[pColList->numOfBound], 0, sizeof(int32_t) * (pColList->numOfCols - pColList->numOfBound)); + + return TSDB_CODE_SUCCESS; +} + +// pSql -> tag1_value, ...) +static int32_t parseTagsClause(SInsertParseContext* pCxt, SParsedDataColInfo* pSpd, SSchema* pTagsSchema, uint8_t precision) { + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + destroyBoundColumnInfo(pSpd); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SKvParam param = {.builder = &kvRowBuilder}; + SToken sToken; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + for (int i = 0; i < pSpd->numOfBound; ++i) { + NEXT_TOKEN(pCxt->pSql, sToken); + SSchema* pSchema = &pTagsSchema[pSpd->boundedColumns[i]]; + param.schema = pSchema; + CHECK_CODE_2(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m), tdDestroyKVRowBuilder(&kvRowBuilder), destroyBoundColumnInfo(pSpd)); + } + + destroyBoundColumnInfo(pSpd); + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + tdDestroyKVRowBuilder(&kvRowBuilder); + if (NULL == row) { + return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); + } + tdSortKVRowByColIdx(row); + + // todo construct payload + + tfree(row); +} + +// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) +static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) { + SToken sToken; + + // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) + NEXT_TOKEN(pCxt->pSql, sToken); + CHECK_CODE(getTableMeta(pCxt, &sToken)); + if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { + return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); + } + + SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); + SParsedDataColInfo spd = {0}; + setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); + + // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_LP == sToken.type) { + CHECK_CODE_1(parseBoundColumns(pCxt, &spd, pTagsSchema), destroyBoundColumnInfo(&spd)); + NEXT_TOKEN(pCxt->pSql, sToken); + } + + if (TK_TAGS != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z); + } + // pSql -> (tag1_value, ...) + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_LP != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); + } + CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); + + return TSDB_CODE_SUCCESS; +} + +static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) { + SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo; + SMemRowBuilder* pBuilder = &pDataBlocks->rowBuilder; + char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header + initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound); + + bool isParseBindParam = false; + SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta); + SMemParam param = {.row = row}; + SToken sToken = {0}; + // 1. set the parsed value from sql string + for (int i = 0; i < spd->numOfBound; ++i) { + NEXT_TOKEN(pCxt->pSql, sToken); + SSchema *pSchema = &schema[spd->boundedColumns[i]]; + param.schema = pSchema; + param.compareStat = pBuilder->compareStat; + getMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, ¶m.toffset); + CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, ¶m)); + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + TSKEY tsKey = memRowKey(row); + if (checkTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) { + buildSyntaxErrMsg(&pCxt->msg, "client time/server time can not be mixed up", sToken.z); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + } + + if (!isParseBindParam) { + // 2. check and set convert flag + if (pBuilder->compareStat == ROW_COMPARE_NEED) { + convertMemRow(row, spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE, pBuilder->kvRowInitLen); + } + + // 3. set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) { + SDataRow dataRow = memRowDataBody(row); + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { + tdAppendDataColVal(dataRow, getNullValue(schema[i].type), true, schema[i].type, spd->cols[i].toffset); + } + } + } + } + + *len = getExtendedRowSize(pDataBlocks); + return TSDB_CODE_SUCCESS; +} + +// pSql -> (field1_value, ...) [(field1_value2, ...) ...] +static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows) { + STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + CHECK_CODE(initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound, pDataBlock->boundColumnInfo.allNullLen)); + + (*numOfRows) = 0; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + SToken sToken; + while (1) { + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_LP != sToken.type) { + break; + } + + if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { + int32_t tSize; + CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize)); + ASSERT(tSize >= maxRows); + maxRows = tSize; + } + + int32_t len = 0; + CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &len, tmpTokenBuf)); + pDataBlock->size += len; + + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_RP != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z); + } + + (*numOfRows)++; + } + + if (0 == (*numOfRows)) { + return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* dataBuf) { + int32_t maxNumOfRows; + CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows)); + + int32_t numOfRows = 0; + CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows)); + + SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows)) { + return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than 32767"); + } + + dataBuf->numOfTables = 1; + pCxt->totalNum += numOfRows; + return TSDB_CODE_SUCCESS; +} + +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +static int32_t parseInsertBody(SInsertParseContext* pCxt) { + while (1) { + SToken sToken; + // pSql -> tb_name ... + NEXT_TOKEN(pCxt->pSql, sToken); + + // no data in the sql string anymore. + if (sToken.n == 0) { + if (0 == pCxt->totalNum) { + return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");; + } + break; + } + + SToken tbnameToken = sToken; + NEXT_TOKEN(pCxt->pSql, sToken); + + // USING cluase + if (TK_USING == sToken.type) { + CHECK_CODE(parseUsingClause(pCxt, &tbnameToken)); + NEXT_TOKEN(pCxt->pSql, sToken); + } else { + CHECK_CODE(getTableMeta(pCxt, &tbnameToken)); + } + + STableDataBlocks *dataBuf = NULL; + CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); + + if (TK_LP == sToken.type) { + // pSql -> field1_name, ...) + CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); + NEXT_TOKEN(pCxt->pSql, sToken); + } + + if (TK_VALUES == sToken.type) { + // pSql -> (field1_value, ...) [(field1_value2, ...) ...] + CHECK_CODE(parseValuesClause(pCxt, dataBuf)); + pCxt->pOutput->insertType = TSDB_QUERY_TYPE_INSERT; + continue; + } + + // FILE csv_file_path + if (TK_FILE == sToken.type) { + // pSql -> csv_file_path + NEXT_TOKEN(pCxt->pSql, sToken); + if (0 == sToken.n || (TK_STRING != sToken.type && TK_ID != sToken.type)) { + return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); + } + // todo + pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT; + continue; + } + + return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); + } + // merge according to vgId + if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, true)); + } + return TSDB_CODE_SUCCESS; +} + +// INSERT INTO +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { + CHECK_CODE(createInsertStmtInfo(pInfo)); + + SInsertParseContext context = { + .pComCxt = pContext, + .pSql = pContext->pSql, + .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, + .pCatalog = NULL, + .pTableMeta = NULL, + .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), + .totalNum = 0, + .pOutput = *pInfo + }; + + CHECK_CODE(catalogGetHandle(NULL, &context.pCatalog)); //TODO + + if (NULL == context.pTableBlockHashObj) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + CHECK_CODE(skipInsertInto(&context)); + CHECK_CODE(parseInsertBody(&context)); + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 548ab1b8df8b5146b808595b9649ca87ca39c40f..d167f7ad3cffe70dc1a17b5a3b56be01ae5451ea 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -18,6 +18,7 @@ #include "parserUtil.h" #include "ttoken.h" #include "function.h" +#include "insertParser.h" bool qIsInsertSql(const char* pStr, size_t length) { int32_t index = 0; @@ -51,8 +52,8 @@ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); } -int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen) { - return 0; +int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { + return parseInsertSql(pContext, pInfo); } int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) { @@ -178,7 +179,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* p assert(t != NULL); if (t->n >= TSDB_FUNC_NAME_LEN) { - return buildSyntaxErrMsg(msg, msgBufLen, "too long function name", t->z); + return buildSyntaxErrMsg(&msgBuf, "too long function name", t->z); } // Let's assume that it is an UDF/UDAF, if it is not a built-in function. diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index f154599aae9fd306438329ab074da4f18b643cba..28d01b9e66f35d38abca57a7fbe9504e61ee9647 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "parserUtil.h" + #include "taosmsg.h" #include "parser.h" #include "taoserror.h" @@ -18,7 +34,6 @@ typedef struct STableFilterCond { static STableMetaInfo* addTableMetaInfo(SQueryStmtInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); -STableMeta* tableMetaDup(STableMeta* pTableMeta); int32_t parserValidateIdToken(SToken* pToken) { if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) { @@ -87,7 +102,7 @@ int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { return TSDB_CODE_TSC_INVALID_OPERATION; } -int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo, const char* sourceStr) { +int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { const char* msgFormat1 = "syntax error near \'%s\'"; const char* msgFormat2 = "syntax error near \'%s\' (%s)"; const char* msgFormat3 = "%s"; @@ -95,7 +110,7 @@ int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalIn const char* prefix = "syntax error"; if (sourceStr == NULL) { assert(additionalInfo != NULL); - snprintf(dst, dstBufLen, msgFormat1, additionalInfo); + snprintf(pBuf->buf, pBuf->len, msgFormat1, additionalInfo); return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } @@ -103,10 +118,10 @@ int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalIn strncpy(buf, sourceStr, tListLen(buf) - 1); if (additionalInfo != NULL) { - snprintf(dst, dstBufLen, msgFormat2, buf, additionalInfo); + snprintf(pBuf->buf, pBuf->len, msgFormat2, buf, additionalInfo); } else { const char* msgFormat = (0 == strncmp(sourceStr, prefix, strlen(prefix))) ? msgFormat3 : msgFormat1; - snprintf(dst, dstBufLen, msgFormat, buf); + snprintf(pBuf->buf, pBuf->len, msgFormat, buf); } return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; @@ -1473,7 +1488,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { return pTableMeta; } -uint32_t getTableMetaSize(STableMeta* pTableMeta) { +uint32_t getTableMetaSize(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); int32_t totalCols = 0; @@ -1488,7 +1503,7 @@ uint32_t getTableMetaMaxSize() { return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); } -STableMeta* tableMetaDup(STableMeta* pTableMeta) { +STableMeta* tableMetaDup(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); size_t size = getTableMetaSize(pTableMeta); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 2d6ec3f864fb26fe169d191f925ee13d2a038529..53a7ba48f84efb9864bb91aed8cc583d3309e2e8 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2312,14 +2312,14 @@ static void yy_reduce( break; case 26: /* cmd ::= SHOW dbPrefix STABLES */ { - setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &yymsp[-1].minor.yy0, 0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &yymsp[-1].minor.yy0, 0); } break; case 27: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { SToken token; tSetDbName(&token, &yymsp[-3].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &yymsp[0].minor.yy0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &token, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= SHOW dbPrefix VGROUPS */ diff --git a/source/libs/parser/src/ttokenizer.c b/source/libs/parser/src/ttokenizer.c index 683fe142ef7b41e4b79eb0bdb52e7bdd4d2fa8cc..d69fa2b95b0feb6fdd8e57f54d34331070baca90 100644 --- a/source/libs/parser/src/ttokenizer.c +++ b/source/libs/parser/src/ttokenizer.c @@ -284,7 +284,7 @@ static int32_t tKeywordCode(const char* z, int n) { * Return the length of the token that begins at z[0]. * Store the token type in *type before returning. */ -uint32_t tGetToken(char* z, uint32_t* tokenId) { +uint32_t tGetToken(const char* z, uint32_t* tokenId) { uint32_t i; switch (*z) { case ' ': @@ -595,7 +595,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) { return ntoken; } -SToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { +SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { SToken t0 = {0}; // here we reach the end of sql string, null-terminated string @@ -689,13 +689,12 @@ void taosCleanupKeywordsTable() { } SToken taosTokenDup(SToken* pToken, char* buf, int32_t len) { - assert(pToken != NULL && buf != NULL); + assert(pToken != NULL && buf != NULL && len > pToken->n); + + strncpy(buf, pToken->z, pToken->n); + buf[pToken->n] = 0; + SToken token = *pToken; token.z = buf; - - assert(len > token.n); - strncpy(token.z, pToken->z, pToken->n); - token.z[token.n] = 0; - return token; } diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9cf48da4eb324727d00708d25fa548b9ef77de48 --- /dev/null +++ b/source/libs/parser/test/insertTest.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "insertParser.h" +#include "mockCatalog.h" + +using namespace std; +using namespace testing; + +namespace { + string toString(int32_t code) { + return tstrerror(code); + } +} + +// syntax: +// INSERT INTO +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +class InsertTest : public Test { +protected: + void setDatabase(const string& db) { + db_ = db; + } + + void bind(const char* sql) { + reset(); + cxt_.sqlLen = strlen(sql); + strcpy(sqlBuf_, sql); + sqlBuf_[cxt_.sqlLen] = '\0'; + cxt_.pSql = sqlBuf_; + cxt_.pDbname = db_.c_str(); + } + + int32_t run() { + code_ = parseInsertSql(&cxt_, &res_); + if (code_ != TSDB_CODE_SUCCESS) { + cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; + } + return code_; + } + + SInsertStmtInfo* reslut() { + return res_; + } + +private: + static const int max_err_len = 1024; + static const int max_sql_len = 1024 * 1024; + + void reset() { + memset(&cxt_, 0, sizeof(cxt_)); + memset(errMagBuf_, 0, max_err_len); + cxt_.pMsg = errMagBuf_; + cxt_.msgLen = max_err_len; + code_ = TSDB_CODE_SUCCESS; + res_ = nullptr; + } + + string db_; + char errMagBuf_[max_err_len]; + char sqlBuf_[max_sql_len]; + SParseContext cxt_; + int32_t code_; + SInsertStmtInfo* res_; +}; + +// INSERT INTO tb_name VALUES (field1_value, ...) +TEST_F(InsertTest, simpleTest) { + setDatabase("test"); + + bind("insert into t1 values (now, 1, \"wxy\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + SInsertStmtInfo* res = reslut(); + // todo check + ASSERT_EQ(res->insertType, TSDB_QUERY_TYPE_INSERT); + // ASSERT_EQ(taosArrayGetSize(res->pDataBlocks), 1); +} + +TEST_F(InsertTest, toleranceTest) { + setDatabase("test"); + + bind("insert into"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); + bind("insert into t"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); +} diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f2b34971efcb1487929fa6274bc6b34001f7053c --- /dev/null +++ b/source/libs/parser/test/mockCatalog.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mockCatalog.h" + +#include + +namespace { + +void generateTestT1(MockCatalogService* mcs) { + ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) + .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + builder.done(); +} + +void generateTestST1(MockCatalogService* mcs) { + ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) + .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) + .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + builder.done(); + mcs->createSubTable("test", "st1", "st1s1", 1); + mcs->createSubTable("test", "st1", "st1s2", 2); +} + +} + +void generateMetaData(MockCatalogService* mcs) { + generateTestT1(mcs); + generateTestST1(mcs); + mcs->showTables(); +} + +struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { + return mockCatalogService->getCatalogHandle(pMgmtEps); +} + +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) { + return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); +} diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h new file mode 100644 index 0000000000000000000000000000000000000000..e60f7727e6804cc6fb6a8959ecb2440745671163 --- /dev/null +++ b/source/libs/parser/test/mockCatalog.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef MOCK_CATALOG_H +#define MOCK_CATALOG_H + +#include "mockCatalogService.h" + +void generateMetaData(MockCatalogService* mcs); + +// mock +struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData); + +#endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp new file mode 100644 index 0000000000000000000000000000000000000000..457f8d88bdb6d54156e1cc8be29e569978350be9 --- /dev/null +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mockCatalogService.h" + +#include +#include +#include + +#include "tname.h" +#include "ttypes.h" + +std::unique_ptr mockCatalogService; + +class TableBuilder : public ITableBuilder { +public: + virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { + assert(colIndex_ < meta_->tableInfo.numOfTags + meta_->tableInfo.numOfColumns); + SSchema* col = meta_->schema + colIndex_; + col->type = type; + col->colId = colIndex_++; + col->bytes = bytes; + strcpy(col->name, name.c_str()); + return *this; + } + + virtual TableBuilder& setVgid(int16_t vgid) { + meta_->vgId = vgid; + return *this; + } + + virtual TableBuilder& setPrecision(uint8_t precision) { + meta_->tableInfo.precision = precision; + return *this; + } + + virtual void done() { + meta_->tableInfo.rowSize = rowsize_; + } + +private: + friend class MockCatalogServiceImpl; + + static std::unique_ptr createTableBuilder(int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { + STableMeta* meta = (STableMeta*)std::calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags)); + if (nullptr == meta) { + throw std::bad_alloc(); + } + meta->tableType = tableType; + meta->tableInfo.numOfTags = numOfTags; + meta->tableInfo.numOfColumns = numOfColumns; + return std::unique_ptr(new TableBuilder(meta)); + } + + TableBuilder(STableMeta* meta) : colIndex_(0), rowsize_(0), meta_(meta) { + } + + STableMeta* table() { + return meta_; + } + + int32_t colIndex_; + int32_t rowsize_; + STableMeta* meta_; +}; + +class MockCatalogServiceImpl { +public: + static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); + + MockCatalogServiceImpl() { + } + + struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const { + return (struct SCatalog*)0x01; + } + + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { + assert(nullptr != pMetaReq && 1 == taosArrayGetSize(pMetaReq->pTableName)); + SName* fullName = (SName*)taosArrayGet(pMetaReq->pTableName, 0); + std::unique_ptr table; + int32_t code = copyTableMeta(fullName->dbname, fullName->tname, &table); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + std::unique_ptr tables((SArray*)taosArrayInit(1, sizeof(STableMeta*))); + if (!tables) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + STableMeta* elem = table.release(); + taosArrayPush(tables.get(), &elem); + pMetaData->pTableMeta = tables.release(); + return TSDB_CODE_SUCCESS; + } + + TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { + builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); + meta_[db][tbname].reset(builder_->table()); + meta_[db][tbname]->uid = id_++; + return *(builder_.get()); + } + + void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { + std::unique_ptr table; + if (TSDB_CODE_SUCCESS != copyTableMeta(db, stbname, &table)) { + throw std::runtime_error("copyTableMeta failed"); + } + meta_[db][tbname].reset(table.release()); + meta_[db][tbname]->uid = id_++; + } + + void showTables() const { + // number of forward fills + #define NOF(n) ((n) / 2) + // number of backward fills + #define NOB(n) ((n) % 2 ? (n) / 2 + 1 : (n) / 2) + // center aligned + #define CA(n, s) std::setw(NOF((n) - (s).length())) << "" << (s) << std::setw(NOB((n) - (s).length())) << "" << "|" + // string field length + #define SFL 20 + // string field header + #define SH(h) CA(SFL, std::string(h)) + // string field + #define SF(n) CA(SFL, n) + // integer field length + #define IFL 10 + // integer field header + #define IH(i) CA(IFL, std::string(i)) + // integer field + #define IF(i) CA(IFL, std::to_string(i)) + // split line + #define SL(sn, in) std::setfill('=') << std::setw((sn) * (SFL + 1) + (in) * (IFL + 1)) << "" << std::setfill(' ') + + for (const auto& db : meta_) { + std::cout << "Databse:" << db.first << std::endl; + std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; + std::cout << SL(3, 1) << std::endl; + for (const auto& table : db.second) { + std::cout << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; + } + std::cout << std::endl; + } + + for (const auto& db : meta_) { + for (const auto& table : db.second) { + std::cout << "Table:" << table.first << std::endl; + std::cout << SH("Field") << SH("Type") << SH("DataType") << IH("Bytes") << std::endl; + std::cout << SL(3, 1) << std::endl; + int16_t numOfTags = table.second->tableInfo.numOfTags; + int16_t numOfFields = numOfTags + table.second->tableInfo.numOfColumns; + for (int16_t i = 0; i < numOfFields; ++i) { + const SSchema* schema = table.second->schema + i; + std::cout << SF(std::string(schema->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(schema->type)) << IF(schema->bytes) << std::endl; + } + std::cout << std::endl; + } + } + } + +private: + typedef std::map > TableMetaCache; + typedef std::map DbMetaCache; + + std::string ttToString(int8_t tableType) const { + switch (tableType) { + case TSDB_SUPER_TABLE: + return "super table"; + case TSDB_CHILD_TABLE: + return "child table"; + case TSDB_NORMAL_TABLE: + return "normal table"; + default: + return "unknown"; + } + } + + std::string pToString(uint8_t precision) const { + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + return "millisecond"; + case TSDB_TIME_PRECISION_MICRO: + return "microsecond"; + case TSDB_TIME_PRECISION_NANO: + return "nanosecond"; + default: + return "unknown"; + } + } + + std::string dtToString(int8_t type) const { + return tDataTypes[type].name; + } + + std::string ftToString(int16_t colid, int16_t numOfTags) const { + return (0 == colid ? "column" : (colid <= numOfTags ? "tag" : "column")); + } + + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { + DbMetaCache::const_iterator it = meta_.find(db); + if (meta_.end() == it) { + return std::shared_ptr(); + } + TableMetaCache::const_iterator tit = it->second.find(tbname); + if (it->second.end() == tit) { + return std::shared_ptr(); + } + return tit->second; + } + + int32_t copyTableMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { + std::shared_ptr src = getTableMeta(db, tbname); + if (!src) { + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); + dst->reset((STableMeta*)std::calloc(1, len)); + if (!dst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(dst->get(), src.get(), len); + return TSDB_CODE_SUCCESS; + } + + uint64_t id_; + std::unique_ptr builder_; + DbMetaCache meta_; +}; + +MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { +} + +MockCatalogService::~MockCatalogService() { +} + +struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) const { + return impl_->getCatalogHandle(pMgmtEps); +} + +int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { + return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); +} + +ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { + return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); +} + +void MockCatalogService::createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { + impl_->createSubTable(db, stbname, tbname, vgid); +} + +void MockCatalogService::showTables() const { + impl_->showTables(); +} \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h new file mode 100644 index 0000000000000000000000000000000000000000..66b439b3e9f46a6cf1d1350d1777e9ee3eee6852 --- /dev/null +++ b/source/libs/parser/test/mockCatalogService.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef MOCK_CATALOG_SERVICE_H +#define MOCK_CATALOG_SERVICE_H + +#include +#include + +#include "catalog.h" + +class ITableBuilder { +public: + ITableBuilder& addTag(const std::string& name, int8_t type) { + return addColumn(name, type, tDataTypes[type].bytes); + } + + ITableBuilder& addTag(const std::string& name, int8_t type, int32_t bytes) { + return addColumn(name, type, bytes); + } + + ITableBuilder& addColumn(const std::string& name, int8_t type) { + return addColumn(name, type, tDataTypes[type].bytes); + } + + virtual ITableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) = 0; + virtual ITableBuilder& setVgid(int16_t vgid) = 0; + virtual ITableBuilder& setPrecision(uint8_t precision) = 0; + virtual void done() = 0; +}; + +class MockCatalogServiceImpl; + +class MockCatalogService { +public: + static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); + + MockCatalogService(); + ~MockCatalogService(); + struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const; + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const; + ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0); + void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); + void showTables() const; + +private: + std::unique_ptr impl_; +}; + +extern std::unique_ptr mockCatalogService; + +#endif // MOCK_CATALOG_SERVICE_H diff --git a/source/libs/parser/test/parserMain.cpp b/source/libs/parser/test/parserMain.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5ec304a006291f280ed6f57459ba80a17e74f6ba --- /dev/null +++ b/source/libs/parser/test/parserMain.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include + +#include "mockCatalog.h" + +class ParserEnv : public testing::Environment { +public: + virtual void SetUp() { + mockCatalogService.reset(new MockCatalogService()); + generateMetaData(mockCatalogService.get()); + } + + virtual void TearDown() { + mockCatalogService.reset(); + } + + ParserEnv() {} + virtual ~ParserEnv() {} +}; + +int main(int argc, char* argv[]) { + testing::AddGlobalTestEnvironment(new ParserEnv()); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 54372ae133b032e1e38c159b5e5424269120c434..23c0aae15f6d3fe2f22dda4046abefc25beb9f69 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -79,11 +79,6 @@ static void _init_tvariant_nchar(SVariant* t) { t->nLen = twcslen(t->wpz); } -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - TEST(testCase, validateToken_test) { char t01[] = "abc"; EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c index 06af8ff6c2294750663e40bbf958980272216950..e49b1d7983ca3eb8ab05e0278d5f0f8c8912c476 100644 --- a/source/libs/sync/src/sync.c +++ b/source/libs/sync/src/sync.c @@ -228,7 +228,7 @@ static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyn } static int syncInitRpcClient(SSyncManager* syncManager) { - char secret[TSDB_KEY_LEN] = "secret"; + char secret[TSDB_PASSWORD_LEN] = "secret"; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "sync-client"; diff --git a/source/libs/transport/inc/rpcLog.h b/source/libs/transport/inc/rpcLog.h index 6c4a281d2c0f0bf85975200c833acbac856dc40c..904680bbe66be06b30b49be727ca522b9b5ed47e 100644 --- a/source/libs/transport/inc/rpcLog.h +++ b/source/libs/transport/inc/rpcLog.h @@ -23,11 +23,10 @@ extern "C" { #include "tlog.h" extern int32_t rpcDebugFlag; -extern int8_t tscEmbedded; -#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} -#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} -#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} +#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", rpcDebugFlag, __VA_ARGS__); }} +#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", rpcDebugFlag, __VA_ARGS__); }} +#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", rpcDebugFlag, __VA_ARGS__); }} #define tInfo(...) { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tDebug(...) { if (rpcDebugFlag & DEBUG_DEBUG) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }} #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }} diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 71e08ea76fc5363a3a9ee82bf3517a6bf528b433..2f710f3efdd0d106dc092e546c5c10f7fc504bbc 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -51,8 +51,8 @@ typedef struct { char user[TSDB_UNI_LEN]; // meter ID char spi; // security parameter index char encrypt; // encrypt algorithm - char secret[TSDB_KEY_LEN]; // secret for the link - char ckey[TSDB_KEY_LEN]; // ciphering key + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key void (*cfp)(void *parent, SRpcMsg *, SEpSet *); int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); @@ -97,8 +97,8 @@ typedef struct SRpcConn { char user[TSDB_UNI_LEN]; // user ID for the link char spi; // security parameter index char encrypt; // encryption, 0:1 - char secret[TSDB_KEY_LEN]; // secret for the link - char ckey[TSDB_KEY_LEN]; // ciphering key + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key char secured; // if set to 1, no authentication uint16_t localPort; // for UDP only uint32_t linkUid; // connection unique ID assigned by client @@ -126,6 +126,8 @@ typedef struct SRpcConn { SRpcReqContext *pContext; // request context } SRpcConn; +static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; + int tsRpcMaxUdpSize = 15000; // bytes int tsProgressTimer = 100; // not configurable @@ -220,17 +222,20 @@ static void rpcFree(void *p) { free(p); } -int32_t rpcInit(void) { - tsProgressTimer = tsRpcTimer/2; - tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer; - tsRpcHeadSize = RPC_MSG_OVERHEAD; +static void rpcInitImp(void) { + tsProgressTimer = tsRpcTimer / 2; + tsRpcMaxRetry = tsRpcMaxTime * 1000 / tsProgressTimer; + tsRpcHeadSize = RPC_MSG_OVERHEAD; tsRpcOverhead = sizeof(SRpcReqContext); tsRpcRefId = taosOpenRef(200, rpcFree); +} +int32_t rpcInit(void) { + pthread_once(&tsRpcInitOnce, rpcInitImp); return 0; } - + void rpcCleanup(void) { taosCloseRef(tsRpcRefId); tsRpcRefId = -1; @@ -698,7 +703,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->linkUid = (uint32_t)((int64_t)pConn + taosGetPid() + (int64_t)pConn->tranId); pConn->spi = pRpc->spi; pConn->encrypt = pRpc->encrypt; - if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN); + if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_PASSWORD_LEN); tDebug("%s %p client connection is allocated, uid:0x%x", pRpc->label, pConn, pConn->linkUid); } @@ -1527,9 +1532,9 @@ static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { int ret = -1; tMD5Init(&context); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Update(&context, (uint8_t *)pMsg, msgLen); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Final(&context); if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; @@ -1541,9 +1546,9 @@ static void rpcBuildAuthHead(void *pMsg, int msgLen, void *pAuth, void *pKey) { T_MD5_CTX context; tMD5Init(&context); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Update(&context, (uint8_t *)pMsg, msgLen); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Final(&context); memcpy(pAuth, context.digest, sizeof(context.digest)); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index f56f240904d6c96dabed87ac421d260f3165097a..e546a87326952e8927c752da409dc6cdc812e985 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -18,6 +18,7 @@ #include "wal.h" #include "compare.h" +#include "tchecksum.h" #ifdef __cplusplus extern "C" { @@ -32,6 +33,13 @@ typedef struct WalFileInfo { int64_t fileSize; } WalFileInfo; +#pragma pack(push,1) +typedef struct WalIdxEntry { + int64_t ver; + int64_t offset; +} WalIdxEntry; +#pragma pack(pop) + static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; WalFileInfo* pInfoRight = (WalFileInfo*)pRight; @@ -72,11 +80,31 @@ static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) { } static inline int walBuildLogName(SWal*pWal, int64_t fileFirstVer, char* buf) { - return sprintf(buf, "%s/%" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); + return sprintf(buf, "%s/%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); } static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { - return sprintf(buf, "%s/%" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); + return sprintf(buf, "%s/%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); +} + +static inline int walValidHeadCksum(SWalHead* pHead) { + return taosCheckChecksum((uint8_t*)&pHead->head, sizeof(SWalReadHead), pHead->cksumHead); +} + +static inline int walValidBodyCksum(SWalHead* pHead) { + return taosCheckChecksum((uint8_t*)pHead->head.body, pHead->head.len, pHead->cksumBody); +} + +static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { + return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); +} + +static inline uint32_t walCalcHeadCksum(SWalHead *pHead) { + return taosCalcChecksum(0, (uint8_t*)&pHead->head, sizeof(SWalReadHead)); +} + +static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) { + return taosCalcChecksum(0, (uint8_t*)body, len); } int walReadMeta(SWal* pWal); @@ -87,6 +115,10 @@ char* walMetaSerialize(SWal* pWal); int walMetaDeserialize(SWal* pWal, const char* bytes); //meta section end +//seek section +int walChangeFile(SWal *pWal, int64_t ver); +//seek section end + int64_t walGetSeq(); int walSeekVer(SWal *pWal, int64_t ver); int walRoll(SWal *pWal); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 65085bb96ddbffe46518d9f9f50ebf1c3162b3bc..49f4fde3a0331ea19b10599b1fd52a2429d979ac 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -24,32 +24,46 @@ #include #include +int64_t walGetFirstVer(SWal *pWal) { + return pWal->vers.firstVer; +} + +int64_t walGetSnaphostVer(SWal *pWal) { + return pWal->vers.snapshotVer; +} + +int64_t walGetLastVer(SWal *pWal) { + return pWal->vers.lastVer; +} + int walRollFileInfo(SWal* pWal) { int64_t ts = taosGetTimestampSec(); SArray* pArray = pWal->fileInfoSet; if(taosArrayGetSize(pArray) != 0) { WalFileInfo *pInfo = taosArrayGetLast(pArray); - pInfo->lastVer = pWal->lastVersion; + pInfo->lastVer = pWal->vers.lastVer; pInfo->closeTs = ts; } + //TODO: change to emplace back WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo)); if(pNewInfo == NULL) { return -1; } - pNewInfo->firstVer = pWal->lastVersion + 1; + pNewInfo->firstVer = pWal->vers.lastVer + 1; pNewInfo->lastVer = -1; pNewInfo->createTs = ts; pNewInfo->closeTs = -1; pNewInfo->fileSize = 0; taosArrayPush(pWal->fileInfoSet, pNewInfo); + free(pNewInfo); return 0; } char* walMetaSerialize(SWal* pWal) { char buf[30]; - if(pWal == NULL || pWal->fileInfoSet == NULL) return 0; + ASSERT(pWal->fileInfoSet); int sz = pWal->fileInfoSet->size; cJSON* pRoot = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject(); @@ -60,13 +74,13 @@ char* walMetaSerialize(SWal* pWal) { return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); - sprintf(buf, "%" PRId64, pWal->firstVersion); + sprintf(buf, "%" PRId64, pWal->vers.firstVer); cJSON_AddStringToObject(pMeta, "firstVer", buf); - sprintf(buf, "%" PRId64, pWal->snapshotVersion); + sprintf(buf, "%" PRId64, pWal->vers.snapshotVer); cJSON_AddStringToObject(pMeta, "snapshotVer", buf); - sprintf(buf, "%" PRId64, pWal->commitVersion); + sprintf(buf, "%" PRId64, pWal->vers.commitVer); cJSON_AddStringToObject(pMeta, "commitVer", buf); - sprintf(buf, "%" PRId64, pWal->lastVersion); + sprintf(buf, "%" PRId64, pWal->vers.lastVer); cJSON_AddStringToObject(pMeta, "lastVer", buf); cJSON_AddItemToObject(pRoot, "files", pFiles); @@ -91,7 +105,9 @@ char* walMetaSerialize(SWal* pWal) { sprintf(buf, "%" PRId64, pInfo->fileSize); cJSON_AddStringToObject(pField, "fileSize", buf); } - return cJSON_Print(pRoot); + char* serialized = cJSON_Print(pRoot); + cJSON_Delete(pRoot); + return serialized; } int walMetaDeserialize(SWal* pWal, const char* bytes) { @@ -100,18 +116,19 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pRoot = cJSON_Parse(bytes); pMeta = cJSON_GetObjectItem(pRoot, "meta"); pField = cJSON_GetObjectItem(pMeta, "firstVer"); - pWal->firstVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); - pWal->snapshotVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "commitVer"); - pWal->commitVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "lastVer"); - pWal->lastVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField)); pFiles = cJSON_GetObjectItem(pRoot, "files"); int sz = cJSON_GetArraySize(pFiles); //deserialize - SArray* pArray = taosArrayInit(sz, sizeof(WalFileInfo)); + SArray* pArray = pWal->fileInfoSet; + taosArrayEnsureCap(pArray, sz); WalFileInfo *pData = pArray->pData; for(int i = 0; i < sz; i++) { cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); @@ -129,6 +146,7 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { } taosArraySetSize(pArray, sz); pWal->fileInfoSet = pArray; + cJSON_Delete(pRoot); return 0; } @@ -143,7 +161,7 @@ static int walFindCurMetaVer(SWal* pWal) { DIR *dir = opendir(pWal->path); if(dir == NULL) { - wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } @@ -159,6 +177,8 @@ static int walFindCurMetaVer(SWal* pWal) { break; } } + closedir(dir); + regfree(&walMetaRegexPattern); return metaVer; } @@ -183,6 +203,7 @@ int walWriteMeta(SWal* pWal) { walBuildMetaName(pWal, metaVer, fnameStr); remove(fnameStr); } + free(serialized); return 0; } @@ -203,6 +224,7 @@ int walReadMeta(SWal* pWal) { if(buf == NULL) { return -1; } + memset(buf, 0, size+5); int tfd = tfOpenRead(fnameStr); if(tfRead(tfd, buf, size) != size) { free(buf); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 6bf90089178370ed187195b0ca62485e21c73d70..629451a7223cb0530f775a550151d3aaf07468eb 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -68,9 +68,12 @@ int32_t walInit() { } void walCleanUp() { + int old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); + if(old == 0) { + return; + } walStopThread(); taosCloseRef(tsWal.refSetId); - atomic_store_8(&tsWal.inited, 0); wInfo("wal module is cleaned up"); } @@ -80,23 +83,30 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } + memset(pWal, 0, sizeof(SWal)); pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; + pWal->writeCur = -1; //set config - pWal->vgId = pCfg->vgId; - pWal->fsyncPeriod = pCfg->fsyncPeriod; - pWal->rollPeriod = pCfg->rollPeriod; - pWal->segSize = pCfg->segSize; - pWal->level = pCfg->walLevel; + memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); + + //init version info + pWal->vers.firstVer = -1; + pWal->vers.commitVer = -1; + pWal->vers.snapshotVer = -1; + pWal->vers.lastVer = -1; + + pWal->vers.verInSnapshotting = -1; + + pWal->totSize = 0; //init status - pWal->lastVersion = -1; pWal->lastRollSeq = -1; //init write buffer - memset(&pWal->head, 0, sizeof(SWalHead)); - pWal->head.sver = 0; + memset(&pWal->writeHead, 0, sizeof(SWalHead)); + pWal->writeHead.head.sver = 0; tstrncpy(pWal->path, path, sizeof(pWal->path)); pthread_mutex_init(&pWal->mutex, NULL); @@ -114,8 +124,9 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walFreeObj(pWal); return NULL; } + walReadMeta(pWal); - wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); + wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); return pWal; } @@ -123,17 +134,17 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { if (pWal == NULL) return TSDB_CODE_WAL_APP_ERROR; - if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) { - wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->vgId, pWal->level, - pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod) { + wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->cfg.vgId, pWal->cfg.level, + pWal->cfg.fsyncPeriod, pCfg->level, pCfg->fsyncPeriod); return 0; } - wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->vgId, pWal->level, - pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->cfg.vgId, pWal->cfg.level, + pWal->cfg.fsyncPeriod, pCfg->level, pCfg->fsyncPeriod); - pWal->level = pCfg->walLevel; - pWal->fsyncPeriod = pCfg->fsyncPeriod; + pWal->cfg.level = pCfg->level; + pWal->cfg.fsyncPeriod = pCfg->fsyncPeriod; pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; @@ -145,31 +156,34 @@ void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); tfClose(pWal->writeLogTfd); + pWal->writeLogTfd = -1; tfClose(pWal->writeIdxTfd); - /*taosArrayDestroy(pWal->fileInfoSet);*/ - /*pWal->fileInfoSet = NULL;*/ + pWal->writeIdxTfd = -1; + walWriteMeta(pWal); + taosArrayDestroy(pWal->fileInfoSet); + pWal->fileInfoSet = NULL; pthread_mutex_unlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } static int32_t walInitObj(SWal *pWal) { if (taosMkDir(pWal->path) != 0) { - wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - pWal->fileInfoSet = taosArrayInit(0, sizeof(WalFileInfo)); + pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); if(pWal->fileInfoSet == NULL) { - wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - wDebug("vgId:%d, object is initialized", pWal->vgId); + wDebug("vgId:%d, object is initialized", pWal->cfg.vgId); return 0; } static void walFreeObj(void *wal) { SWal *pWal = wal; - wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); + wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); tfClose(pWal->writeLogTfd); tfClose(pWal->writeIdxTfd); @@ -180,7 +194,7 @@ static void walFreeObj(void *wal) { } static bool walNeedFsync(SWal *pWal) { - if (pWal->fsyncPeriod <= 0 || pWal->level != TAOS_WAL_FSYNC) { + if (pWal->cfg.fsyncPeriod <= 0 || pWal->cfg.level != TAOS_WAL_FSYNC) { return false; } @@ -200,10 +214,10 @@ static void walFsyncAll() { SWal *pWal = taosIterateRef(tsWal.refSetId, 0); while (pWal) { if (walNeedFsync(pWal)) { - wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); int32_t code = tfFsync(pWal->writeLogTfd); if (code != 0) { - wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(code)); + wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); } } pWal = taosIterateRef(tsWal.refSetId, pWal->refId); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index e9f5bcbc5de00439a06e588f190d1110c2a2144a..b6aafedea3b4ac9150370d7012407c34a9c7e311 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -15,18 +15,162 @@ #include "walInt.h" #include "tfile.h" -#include "tchecksum.h" -static inline int walValidHeadCksum(SWalHead* pHead) { - return taosCheckChecksum((uint8_t*)pHead, sizeof(SWalHead) - sizeof(uint32_t)*2, pHead->cksumHead); +SWalReadHandle* walOpenReadHandle(SWal* pWal) { + SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); + if(pRead == NULL) { + return NULL; + } + pRead->pWal = pWal; + pRead->readIdxTfd = -1; + pRead->readLogTfd = -1; + pRead->curVersion = -1; + pRead->curFileFirstVer = -1; + pRead->capacity = 0; + pRead->status = 0; + pRead->pHead = malloc(sizeof(SWalHead)); + if(pRead->pHead == NULL) { + free(pRead); + return NULL; + } + return pRead; +} + +void walCloseReadHandle(SWalReadHandle *pRead) { + tfClose(pRead->readIdxTfd); + tfClose(pRead->readLogTfd); + tfree(pRead->pHead); + free(pRead); +} + +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { + return 0; +} + +static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { + int code = 0; + + int64_t idxTfd = pRead->readIdxTfd; + int64_t logTfd = pRead->readLogTfd; + + //seek position + int64_t offset = (ver - fileFirstVer) * WAL_IDX_ENTRY_SIZE; + code = tfLseek(idxTfd, offset, SEEK_SET); + if(code < 0) { + return -1; + } + WalIdxEntry entry; + if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + return -1; + } + //TODO:deserialize + ASSERT(entry.ver == ver); + code = tfLseek(logTfd, entry.offset, SEEK_SET); + if (code < 0) { + return -1; + } + return code; +} + +static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { + char fnameStr[WAL_FILE_LEN]; + + tfClose(pRead->readIdxTfd); + tfClose(pRead->readLogTfd); + + walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); + int64_t logTfd = tfOpenRead(fnameStr); + if(logTfd < 0) { + return -1; + } + + walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); + int64_t idxTfd = tfOpenRead(fnameStr); + if(idxTfd < 0) { + return -1; + } + + pRead->readLogTfd = logTfd; + pRead->readIdxTfd = idxTfd; + return 0; } -static inline int walValidBodyCksum(SWalHead* pHead) { - return taosCheckChecksum((uint8_t*)pHead->cont, pHead->len, pHead->cksumBody); +static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { + int code; + SWal *pWal = pRead->pWal; + if(ver == pRead->curVersion) { + return 0; + } + if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { + return -1; + } + if(ver < pWal->vers.snapshotVer) { + + } + + WalFileInfo tmpInfo; + tmpInfo.firstVer = ver; + //bsearch in fileSet + WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + ASSERT(pRet != NULL); + if(pRead->curFileFirstVer != pRet->firstVer) { + code = walReadChangeFile(pRead, pRet->firstVer); + if(code < 0) { + //TODO: set error flag + return -1; + } + } + + code = walReadSeekFilePos(pRead, pRet->firstVer, ver); + if(code < 0) { + return -1; + } + pRead->curVersion = ver; + + return 0; } -static int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { - return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); +int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { + int code; + //TODO: check wal life + if(pRead->curVersion != ver) { + code = walReadSeekVer(pRead, ver); + if(code != 0) { + return -1; + } + } + + if(!tfValid(pRead->readLogTfd)) return -1; + + code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + if(code != sizeof(SWalHead)) { + return -1; + } + code = walValidHeadCksum(pRead->pHead); + if(code != 0) { + return -1; + } + if(pRead->capacity < pRead->pHead->head.len) { + void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); + if(ptr == NULL) { + return -1; + } + pRead->pHead = ptr; + pRead->capacity = pRead->pHead->head.len; + } + if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + return -1; + } + + /*code = walValidBodyCksum(pRead->pHead);*/ + ASSERT(pRead->pHead->head.version == ver); + + if(code != 0) { + return -1; + } + pRead->curVersion++; + + return 0; } int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { @@ -49,13 +193,13 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { if(walValidHeadCksum(*ppHead) != 0) { return -1; } - void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->len); + void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); if(ptr == NULL) { free(*ppHead); *ppHead = NULL; return -1; } - if(tfRead(pWal->writeLogTfd, (*ppHead)->cont, (*ppHead)->len) != (*ppHead)->len) { + if(tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } //TODO: endian compatibility processing after read @@ -69,18 +213,3 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) { return 0; } - -int64_t walGetFirstVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->firstVersion; -} - -int64_t walGetSnaphostVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->snapshotVersion; -} - -int64_t walGetLastVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->lastVersion; -} diff --git a/source/libs/wal/src/walIndex.c b/source/libs/wal/src/walSeek.c similarity index 76% rename from source/libs/wal/src/walIndex.c rename to source/libs/wal/src/walSeek.c index bf51be434681ebb05fcb9a202f36708f73c5d256..953aae703c75764acf047851022b15c79b88cc77 100644 --- a/source/libs/wal/src/walIndex.c +++ b/source/libs/wal/src/walSeek.c @@ -43,22 +43,47 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { if (code != 0) { return -1; } - /*pWal->curLogOffset = readBuf[1];*/ - pWal->curVersion = ver; return code; } -static int walChangeFile(SWal *pWal, int64_t ver) { +int walChangeFileToLast(SWal *pWal) { + int64_t idxTfd, logTfd; + WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); + ASSERT(pRet != NULL); + int64_t fileFirstVer = pRet->firstVer; + + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, fileFirstVer, fnameStr); + idxTfd = tfOpenReadWrite(fnameStr); + if(idxTfd < 0) { + return -1; + } + walBuildLogName(pWal, fileFirstVer, fnameStr); + logTfd = tfOpenReadWrite(fnameStr); + if(logTfd < 0) { + return -1; + } + //switch file + pWal->writeIdxTfd = idxTfd; + pWal->writeLogTfd = logTfd; + //change status + pWal->curStatus = WAL_CUR_FILE_WRITABLE; + return 0; +} + +int walChangeFile(SWal *pWal, int64_t ver) { int code = 0; int64_t idxTfd, logTfd; char fnameStr[WAL_FILE_LEN]; code = tfClose(pWal->writeLogTfd); if(code != 0) { //TODO + return -1; } code = tfClose(pWal->writeIdxTfd); if(code != 0) { //TODO + return -1; } WalFileInfo tmpInfo; tmpInfo.firstVer = ver; @@ -83,24 +108,19 @@ static int walChangeFile(SWal *pWal, int64_t ver) { pWal->writeLogTfd = logTfd; pWal->writeIdxTfd = idxTfd; - return code; + return fileFirstVer; } int walSeekVer(SWal *pWal, int64_t ver) { int code; - if((!(pWal->curStatus & WAL_CUR_FAILED)) && ver == pWal->curVersion) { + if(ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->lastVersion) { - //TODO: some records are skipped + if(ver > pWal->vers.lastVer|| ver < pWal->vers.firstVer) { return -1; } - if(ver < pWal->firstVersion) { - //TODO: try to seek pruned log - return -1; - } - if(ver < pWal->snapshotVersion) { - //TODO: seek snapshotted log, invalid in some cases + if(ver < pWal->vers.snapshotVer) { + } if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { code = walChangeFile(pWal, ver); diff --git a/source/libs/wal/src/walUtil.c b/source/libs/wal/src/walUtil.c index c88cc918fe42c30564abe3feb5ae273386d1b994..849d0c3e5189acdd6fdd50a6ef3210351ffbb64d 100644 --- a/source/libs/wal/src/walUtil.c +++ b/source/libs/wal/src/walUtil.c @@ -17,6 +17,7 @@ #include "os.h" #include "walInt.h" +#if 0 int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId) { int64_t curFileId = *nextFileId; int64_t minFileId = INT64_MAX; @@ -116,3 +117,4 @@ int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) { return 0; } +#endif diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 0c4989300f1c14c57395429f7f1a88d703893772..994b8fc33323a001578bb38b0741a3a7c0eac5cf 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -21,65 +21,6 @@ #include "tfile.h" #include "walInt.h" -static void walFtruncate(SWal *pWal, int64_t ver); - -int32_t walCommit(SWal *pWal, int64_t ver) { - ASSERT(pWal->snapshotVersion <= pWal->commitVersion); - ASSERT(pWal->commitVersion <= pWal->lastVersion); - ASSERT(ver >= pWal->commitVersion); - ASSERT(ver <= pWal->lastVersion); - pWal->commitVersion = ver; - return 0; -} - -int32_t walRollback(SWal *pWal, int64_t ver) { - //TODO: ftruncate - ASSERT(ver > pWal->commitVersion); - ASSERT(ver <= pWal->lastVersion); - //seek position - walSeekVer(pWal, ver); - walFtruncate(pWal, ver); - return 0; -} - -int32_t walTakeSnapshot(SWal *pWal, int64_t ver) { - pWal->snapshotVersion = ver; - int ts = taosGetTimestampSec(); - - int deleteCnt = 0; - int64_t newTotSize = pWal->totSize; - WalFileInfo tmp; - tmp.firstVer = ver; - //mark files safe to delete - WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); - //iterate files, until the searched result - for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if(pWal->totSize > pWal->retentionSize || - iter->closeTs + pWal->retentionPeriod > ts) { - //delete according to file size or close time - deleteCnt++; - newTotSize -= iter->fileSize; - } - } - char fnameStr[WAL_FILE_LEN]; - //remove file - for(int i = 0; i < deleteCnt; i++) { - WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); - walBuildLogName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); - walBuildIdxName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); - } - - //save snapshot ver, commit ver - - - //make new array, remove files - taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); - pWal->totSize = newTotSize; - - return 0; -} #if 0 static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId); @@ -172,6 +113,188 @@ void walRemoveAllOldFiles(void *handle) { } #endif +int32_t walCommit(SWal *pWal, int64_t ver) { + ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); + ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); + if(ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { + return -1; + } + pWal->vers.commitVer = ver; + return 0; +} + +int32_t walRollback(SWal *pWal, int64_t ver) { + int code; + char fnameStr[WAL_FILE_LEN]; + if(ver == pWal->vers.lastVer) { + return 0; + } + if(ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { + return -1; + } + pthread_mutex_lock(&pWal->mutex); + + //find correct file + if(ver < walGetLastFileFirstVer(pWal)) { + //close current files + tfClose(pWal->writeIdxTfd); + tfClose(pWal->writeLogTfd); + //open old files + code = walChangeFile(pWal, ver); + if(code != 0) { + return -1; + } + + //delete files + int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); + for(int i = pWal->writeCur; i < fileSetSize; i++) { + walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + remove(fnameStr); + walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + remove(fnameStr); + } + //pop from fileInfoSet + taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); + } + + walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); + int64_t idxTfd = tfOpenReadWrite(fnameStr); + + //change to deserialize function + + if(idxTfd < 0) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + int idxOff = (ver - walGetCurFileFirstVer(pWal)) * WAL_IDX_ENTRY_SIZE; + code = tfLseek(idxTfd, idxOff, SEEK_SET); + if(code < 0) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + //read idx file and get log file pos + //TODO:change to deserialize function + WalIdxEntry entry; + if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + ASSERT(entry.ver == ver); + + walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); + int64_t logTfd = tfOpenReadWrite(fnameStr); + if(logTfd < 0) { + //TODO + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + code = tfLseek(logTfd, entry.offset, SEEK_SET); + if(code < 0) { + //TODO + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + //validate offset + SWalHead head; + ASSERT(tfValid(logTfd)); + int size = tfRead(logTfd, &head, sizeof(SWalHead)); + if(size != sizeof(SWalHead)) { + return -1; + } + code = walValidHeadCksum(&head); + + ASSERT(code == 0); + if(code != 0) { + return -1; + } + if(head.head.version != ver) { + //TODO + return -1; + } + //truncate old files + code = tfFtruncate(logTfd, entry.offset); + if(code < 0) { + return -1; + } + code = tfFtruncate(idxTfd, idxOff); + if(code < 0) { + return -1; + } + pWal->vers.lastVer = ver - 1; + ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; + ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + + //unlock + pthread_mutex_unlock(&pWal->mutex); + return 0; +} + +int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { + pWal->vers.verInSnapshotting = ver; + //check file rolling + if(pWal->cfg.retentionPeriod == 0) { + walRoll(pWal); + } + + return 0; +} + +int32_t walEndTakeSnapshot(SWal *pWal) { + int64_t ver = pWal->vers.verInSnapshotting; + if(ver == -1) return -1; + + pWal->vers.snapshotVer = ver; + int ts = taosGetTimestampSec(); + + int deleteCnt = 0; + int64_t newTotSize = pWal->totSize; + WalFileInfo tmp; + tmp.firstVer = ver; + //find files safe to delete + WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); + if(ver >= pInfo->lastVer) { + pInfo++; + } + //iterate files, until the searched result + for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { + if(pWal->totSize > pWal->cfg.retentionSize || + iter->closeTs + pWal->cfg.retentionPeriod > ts) { + //delete according to file size or close time + deleteCnt++; + newTotSize -= iter->fileSize; + } + } + char fnameStr[WAL_FILE_LEN]; + //remove file + for(int i = 0; i < deleteCnt; i++) { + WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); + walBuildLogName(pWal, pInfo->firstVer, fnameStr); + remove(fnameStr); + walBuildIdxName(pWal, pInfo->firstVer, fnameStr); + remove(fnameStr); + } + + //make new array, remove files + taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); + if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + pWal->writeCur = -1; + pWal->vers.firstVer = -1; + } else { + pWal->vers.firstVer = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + } + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;; + pWal->totSize = newTotSize; + pWal->vers.verInSnapshotting = -1; + + //save snapshot ver, commit ver + int code = walWriteMeta(pWal); + if(code != 0) { + return -1; + } + + return 0; +} + int walRoll(SWal *pWal) { int code = 0; if(pWal->writeIdxTfd != -1) { @@ -188,7 +311,7 @@ int walRoll(SWal *pWal) { } int64_t idxTfd, logTfd; //create new file - int64_t newFileFirstVersion = pWal->lastVersion + 1; + int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); idxTfd = tfOpenCreateWrite(fnameStr); @@ -211,6 +334,7 @@ int walRoll(SWal *pWal) { //switch file pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; //change status pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE; @@ -218,44 +342,11 @@ int walRoll(SWal *pWal) { return 0; } -int walChangeFileToLast(SWal *pWal) { - int64_t idxTfd, logTfd; - WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); - ASSERT(pRet != NULL); - int64_t fileFirstVer = pRet->firstVer; - - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenReadWrite(fnameStr); - if(idxTfd < 0) { - return -1; - } - walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenReadWrite(fnameStr); - if(logTfd < 0) { - return -1; - } - //switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; - //change status - pWal->curVersion = fileFirstVer; - pWal->curStatus = WAL_CUR_FILE_WRITABLE; - return 0; -} - static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { - int code = 0; - //get index file - if(!tfValid(pWal->writeIdxTfd)) { - code = TAOS_SYSTEM_ERROR(errno); - - wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); - return code; - } - int64_t writeBuf[2] = { ver, offset }; - int size = tfWrite(pWal->writeIdxTfd, writeBuf, sizeof(writeBuf)); - if(size != sizeof(writeBuf)) { + WalIdxEntry entry = { .ver = ver, .offset = offset }; + int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); + if(size != sizeof(WalIdxEntry)) { + //TODO truncate return -1; } return 0; @@ -266,17 +357,18 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i int code = 0; // no wal - if (pWal->level == TAOS_WAL_NOLOG) return 0; + if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0; - if (index == pWal->lastVersion + 1) { + if (index == pWal->vers.lastVer + 1) { if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + pWal->vers.firstVer = index; code = walRoll(pWal); ASSERT(code == 0); } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; - if(pWal->rollPeriod != -1 && passed > pWal->rollPeriod) { + if(pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { walRoll(pWal); - } else if(pWal->segSize != -1 && walGetLastFileSize(pWal) > pWal->segSize) { + } else if(pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { walRoll(pWal); } } @@ -285,37 +377,36 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i //must truncate explicitly first return -1; } - /*if (!tfValid(pWal->curLogTfd)) return 0;*/ - - pWal->head.version = index; - - pWal->head.signature = WAL_SIGNATURE; - pWal->head.len = bodyLen; - pWal->head.msgType = msgType; - - pWal->head.cksumHead = taosCalcChecksum(0, (const uint8_t*)&pWal->head, sizeof(SWalHead)- sizeof(uint32_t)*2); - pWal->head.cksumBody = taosCalcChecksum(0, (const uint8_t*)&body, bodyLen); + /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ pthread_mutex_lock(&pWal->mutex); + pWal->writeHead.head.version = index; + + int64_t offset = walGetCurFileOffset(pWal); + pWal->writeHead.head.len = bodyLen; + pWal->writeHead.head.msgType = msgType; + pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); + pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - if (tfWrite(pWal->writeLogTfd, &pWal->head, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, &body, bodyLen) != bodyLen) { + if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal)); + code = walWriteIndex(pWal, index, offset); if(code != 0) { //TODO + return -1; } //set status - pWal->lastVersion = index; + pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalHead) + bodyLen; walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; @@ -326,12 +417,10 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } void walFsync(SWal *pWal, bool forceFsync) { - if (pWal == NULL || !tfValid(pWal->writeLogTfd)) return; - - if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { - wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->vgId, walGetCurFileFirstVer(pWal)); + if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { + wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (tfFsync(pWal->writeLogTfd) < 0) { - wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->vgId, walGetCurFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } } } @@ -404,38 +493,29 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { } #endif -static int walValidateOffset(SWal* pWal, int64_t ver) { - int code = 0; - SWalHead *pHead = NULL; - code = (int)walRead(pWal, &pHead, ver); - if(pHead->version != ver) { - return -1; - } - return 0; -} - -static int64_t walGetOffset(SWal* pWal, int64_t ver) { - int code = walSeekVer(pWal, ver); - if(code != 0) { - return -1; - } +/*static int walValidateOffset(SWal* pWal, int64_t ver) {*/ + /*int code = 0;*/ + /*SWalHead *pHead = NULL;*/ + /*code = (int)walRead(pWal, &pHead, ver);*/ + /*if(pHead->head.version != ver) {*/ + /*return -1;*/ + /*}*/ + /*return 0;*/ +/*}*/ - code = walValidateOffset(pWal, ver); - if(code != 0) { - return -1; - } +/*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/ + /*int code = walSeekVer(pWal, ver);*/ + /*if(code != 0) {*/ + /*return -1;*/ + /*}*/ - return 0; -} + /*code = walValidateOffset(pWal, ver);*/ + /*if(code != 0) {*/ + /*return -1;*/ + /*}*/ -static void walFtruncate(SWal *pWal, int64_t ver) { - int64_t tfd = pWal->writeLogTfd; - tfFtruncate(tfd, ver); - tfFsync(tfd); - tfd = pWal->writeIdxTfd; - tfFtruncate(tfd, ver * WAL_IDX_ENTRY_SIZE); - tfFsync(tfd); -} + /*return 0;*/ +/*}*/ #if 0 static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) { diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 96258662a1ed58e0ba532b058569c86aa86ed2b0..200bf39c5a087af9edd7cfefea0ed0a5e1c846e6 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -5,6 +5,9 @@ #include "walInt.h" +const char* ranStr = "tvapq02tcp"; +const int ranStrLen = strlen(ranStr); + class WalCleanEnv : public ::testing::Test { protected: static void SetUpTestCase() { @@ -18,12 +21,47 @@ class WalCleanEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; - pCfg->walLevel = TAOS_WAL_FSYNC; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); + free(pCfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; +}; + +class WalCleanDeleteEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } + + static void TearDownTestCase() { + walCleanUp(); + } + + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + free(pCfg); ASSERT(pWal != NULL); } @@ -47,13 +85,22 @@ class WalKeepEnv : public ::testing::Test { walCleanUp(); } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; - pCfg->walLevel = TAOS_WAL_FSYNC; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); + free(pCfg); ASSERT(pWal != NULL); } @@ -94,6 +141,7 @@ TEST_F(WalCleanEnv, serialize) { ASSERT(code == 0); char*ss = walMetaSerialize(pWal); printf("%s\n", ss); + free(ss); code = walWriteMeta(pWal); ASSERT(code == 0); } @@ -111,20 +159,25 @@ TEST_F(WalCleanEnv, removeOldMeta) { } TEST_F(WalKeepEnv, readOldMeta) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walWriteMeta(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walWriteMeta(pWal); - ASSERT(code == 0); - char*oldss = walMetaSerialize(pWal); + walResetEnv(); + int code; + + for(int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, -1); + ASSERT_EQ(pWal->vers.lastVer, i); + } + char* oldss = walMetaSerialize(pWal); TearDown(); SetUp(); - code = walReadMeta(pWal); - ASSERT(code == 0); + + ASSERT_EQ(pWal->vers.firstVer, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + char* newss = walMetaSerialize(pWal); int len = strlen(oldss); @@ -132,18 +185,107 @@ TEST_F(WalKeepEnv, readOldMeta) { for(int i = 0; i < len; i++) { EXPECT_EQ(oldss[i], newss[i]); } + free(oldss); + free(newss); } -TEST_F(WalKeepEnv, write) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); +TEST_F(WalCleanEnv, write) { int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); - code = walWrite(pWal, i+2, i, (void*)ranStr, len); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); + ASSERT_EQ(pWal->vers.lastVer, i); + } + code = walWriteMeta(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalCleanEnv, rollback) { + int code; + for(int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); } + code = walRollback(pWal, 5); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 4); + code = walRollback(pWal, 3); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 2); code = walWriteMeta(pWal); ASSERT_EQ(code, 0); } + +TEST_F(WalCleanDeleteEnv, roll) { + int code; + int i; + for(i = 0; i < 100; i++) { + code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->vers.commitVer, i); + } + + walBeginTakeSnapshot(pWal, i-1); + ASSERT_EQ(pWal->vers.verInSnapshotting, i-1); + walEndTakeSnapshot(pWal); + ASSERT_EQ(pWal->vers.snapshotVer, i-1); + ASSERT_EQ(pWal->vers.verInSnapshotting, -1); + + code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen); + ASSERT_NE(code, 0); + + for(; i < 200; i++) { + code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->vers.commitVer, i); + } + + code = walBeginTakeSnapshot(pWal, i - 1); + ASSERT_EQ(code, 0); + code = walEndTakeSnapshot(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalKeepEnv, readHandleRead) { + walResetEnv(); + int code; + SWalReadHandle* pRead = walOpenReadHandle(pWal); + ASSERT(pRead != NULL); + + int i ; + for(i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walWrite(pWal, i, 0, newStr, len); + ASSERT_EQ(code, 0); + } + for(int i = 0; i < 1000; i++) { + int ver = rand() % 100; + code = walReadWithHandle(pRead, ver); + ASSERT_EQ(code, 0); + + //printf("rrbody: \n"); + //for(int i = 0; i < pRead->pHead->head.len; i++) { + //printf("%d ", pRead->pHead->head.body[i]); + //} + //printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver+1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.len, len); + for(int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } +} diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 2385d102858393ef426ac54d4999c413a3f639a5..0d7066b5c8eac3bc7e312651ef0b1888410d62f2 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -42,7 +42,7 @@ bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == s int32_t taosGetPId() { return GetCurrentProcessId(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { +int32_t taosGetAppName(char* name, int32_t* len) { char filepath[1024] = {0}; GetModuleFileName(NULL, filepath, MAX_PATH); @@ -358,7 +358,7 @@ bool taosComparePthread(pthread_t first, pthread_t second) { return pthread_equa int32_t taosGetPId() { return (int32_t)getpid(); } -int32_t taosGetCurrentAPPName(char *name, int32_t *len) { +int32_t taosGetAppName(char *name, int32_t *len) { char buf[PATH_MAX + 1]; buf[0] = '\0'; proc_name(getpid(), buf, sizeof(buf) - 1); @@ -392,7 +392,7 @@ void taosResetPthread(pthread_t* thread) { *thread = 0; } bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } int32_t taosGetPId() { return getpid(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { +int32_t taosGetAppName(char* name, int32_t* len) { const char* self = "/proc/self/exe"; char path[PATH_MAX] = {0}; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 8054dc42be290a24657fe67531d192cdf5653a14..f714fd0dc4d26cf9f5c1391b34c6277b971cd74d 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -37,7 +37,7 @@ char *taosCharsetReplace(char *charsetstr) { return strdup(charsetstr); } -int64_t taosStr2int64(char *str) { +int64_t taosStr2int64(const char *str) { char *endptr = NULL; return strtoll(str, &endptr, 10); } @@ -107,7 +107,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { return len; } -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { +bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { memset(ucs4, 0, ucs4_max_len); mbstate_t state = {0}; int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); @@ -275,13 +275,13 @@ char *strsep(char **stringp, const char *delim) { } char *getpass(const char *prefix) { - static char passwd[TSDB_KEY_LEN] = {0}; - memset(passwd, 0, TSDB_KEY_LEN); + static char passwd[TSDB_PASSWORD_LEN] = {0}; + memset(passwd, 0, TSDB_PASSWORD_LEN); //printf("%s", prefix); int32_t index = 0; char ch; - while (index < TSDB_KEY_LEN) { + while (index < TSDB_PASSWORD_LEN) { ch = getch(); if (ch == '\n' || ch == '\r') { break; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index ca817c4c1e809bd3a74f2385167403de584f9da8..f892b4d8c0c95c0377274870110f16613012ef3d 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -714,7 +714,7 @@ static void taosGetSystemLocale() { // get and set default locale //printf("locale not configured, set to system default:%s", tsLocale); } - /* if user does not specify the charset, extract it from locale */ + // if user does not specify the charset, extract it from locale char *str = strrchr(tsLocale, sep); if (str != NULL) { str++; @@ -1118,13 +1118,13 @@ char *taosGetCmdlineByPID(int pid) { SysNameInfo taosGetSysNameInfo() { SysNameInfo info = {0}; - struct utsname buf; - if (!uname(&buf)) { - info.sysname = buf.sysname; - info.sysname == buf.nodename; - info.sysname = buf.release; - info.sysname = buf.version; - info.sysname = buf.machine; + struct utsname uts; + if (!uname(&uts)) { + info.sysname = strdup(uts.sysname); + info.nodename = strdup(uts.nodename); + info.release = strdup(uts.release); + info.version = strdup(uts.version); + info.machine = strdup(uts.machine); } return info; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index ff52477a6fb6b769dab93a0d4a3e09e572f31d5e..581a7973432edaf20f8ff986816abd40bd12d816 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -58,24 +58,31 @@ static int32_t taosArrayResize(SArray* pArray) { return 0; } -void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { - if (pArray == NULL || pData == NULL) { - return NULL; - } - - if (pArray->size + nEles > pArray->capacity) { +int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { + if (newCap > pArray->capacity) { size_t tsize = (pArray->capacity << 1u); - while (pArray->size + nEles > tsize) { + while (newCap > tsize) { tsize = (tsize << 1u); } pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize); if (pArray->pData == NULL) { - return NULL; + return -1; } pArray->capacity = tsize; } + return 0; +} + +void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { + if (pArray == NULL || pData == NULL) { + return NULL; + } + + if(taosArrayEnsureCap(pArray, pArray->size + nEles) != 0){ + return NULL; + } void* dst = TARRAY_GET_ELEM(pArray, pArray->size); memcpy(dst, pData, pArray->elemSize * nEles); @@ -241,10 +248,14 @@ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { assert(cnt <= pArray->size); pArray->size = pArray->size - cnt; if(pArray->size == 0) { - pArray->size = 0; return; } - memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size); + memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size * pArray->elemSize); +} + +void taosArrayPopTailBatch(SArray* pArray, size_t cnt) { + assert(cnt <= pArray->size); + pArray->size = pArray->size - cnt; } void taosArrayRemove(SArray* pArray, size_t index) { @@ -329,6 +340,11 @@ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t compa return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); } +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { + void* item = taosArraySearch(pArray, key, comparFn, flags); + return (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize; +} + void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { assert(pArray != NULL); qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8a6f389366d2393c4b38dd499755f9484f694b25..726247d4506c65d8a8f1597a61e8ef0e5c9002c2 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -282,7 +282,7 @@ static void taosReadConfigOption(const char *option, char *value, char *value2, } } -void taosInitConfigOption(SGlobalCfg cfg) { +void taosAddConfigOption(SGlobalCfg cfg) { tsGlobalConfig[tsGlobalConfigNum++] = cfg; } @@ -335,7 +335,7 @@ void taosReadGlobalLogCfg() { fclose(fp); } -int32_t taosReadGlobalCfg() { +int32_t taosReadCfgFromFile() { char * line, *option, *value, *value2, *value3; int olen, vlen, vlen2, vlen3; char fileName[PATH_MAX] = {0}; @@ -396,7 +396,7 @@ int32_t taosReadGlobalCfg() { return 0; } -void taosPrintGlobalCfg() { +void taosPrintCfg() { uInfo(" taos config & system info:"); uInfo("=================================="); @@ -443,7 +443,6 @@ void taosPrintGlobalCfg() { } taosPrintOsInfo(); - // taosPrintDataDirCfg(); uInfo("=================================="); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index db94d265185f77aef3f5482b0aa578d0f0475f65..42fde042e7a56e94b9afb7c6e57015a698614c13 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -163,22 +163,23 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_VER, "Invalid raw data vers TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_LEN, "Invalid raw data len") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_CONTENT, "Invalid raw data content") +// mnode-dnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "DNode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "DNode does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, "Master DNode cannot be removed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of DNodes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, "Cluster cfg inconsistent") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, "Invalid dnode cfg option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED, "Balance already enabled") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_CFG, "Cluster cfg inconsistent") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, "Cluster id not match") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG, "Invalid dnode cfg") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_EP, "Invalid dnode end point") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID, "Invalid dnode id") + +// mnode-vgroup +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE, "Dnode not avaliable") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, "Cluster id not match") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED, "Dnode Id not configured") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED, "Dnode Ep not configured") +// mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_NOT_EXIST, "Invalid account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, "Invalid account options") @@ -194,8 +195,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST, "Table already exists") +// mnode-stable +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 23b66af0c899b01f4b5682f6d66e2e687bf649df..2d7179c0d319c8abad0a58a89e4456cb3fce33b1 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -114,7 +114,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); -extern void taosPrintGlobalCfg(); +extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { @@ -224,7 +224,7 @@ static void *taosThreadToOpenNewFile(void *param) { uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintGlobalCfg(); + taosPrintCfg(); taosKeepOldLog(keepName); return NULL; diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c index 5606ab248d71b62f9b6041b76069a5292cd33223..d9356b7e4049cfdf9bc77ff1606e24993758bd6c 100644 --- a/source/util/src/tnote.c +++ b/source/util/src/tnote.c @@ -49,15 +49,6 @@ int32_t taosInitNotes() { taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); } - if (tsHttpEnableRecordSql) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/httpsql", tsLogDir); - taosInitNote(tsNumOfLogLines, 1, &tsHttpNote, name); - } - - if (tscEmbedded == 1) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/taosinfo", tsLogDir); - taosInitNote(tsNumOfLogLines, 1, &tsInfoNote, name); - } #endif return 0; } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 56186d9b24d5c6763f0c36714a030bc1abf71c1a..1fdc2257d787124d2bff5c9f23bc740bce922314 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -20,12 +20,10 @@ #include "tutil.h" #include "taoserror.h" -extern int8_t tscEmbedded; - -#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} +#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} #define tmrDebug(...) { if (tmrDebugFlag & DEBUG_DEBUG) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} #define tmrTrace(...) { if (tmrDebugFlag & DEBUG_TRACE) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 3c3c8a1dfcef528ca5f81382f986665b6adc5995..58a2e57f7c527cf8e75e8e0d3acf5beab0c4e6d7 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -166,7 +166,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { return split; } -char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) { +char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) { for (int32_t i = 0; i < len; ++i) { // skip the needle in quote, jump to the end of quoted string @@ -179,7 +179,7 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) { } if (haystack[i] == needle) { - return &haystack[i]; + return (char *)&haystack[i]; } } @@ -417,16 +417,3 @@ void taosIp2String(uint32_t ip, char *str) { void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); } - -int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { - *port = 0; - strcpy(fqdn, ep); - - char *temp = strchr(fqdn, ':'); - if (temp) { - *temp = 0; - *port = atoi(temp + 1); - } - - return 0; -} \ No newline at end of file diff --git a/src/client/inc/tscParseLine.h b/src/client/inc/tscParseLine.h index 401dcafdfbefd28e79ebdf30d810e194564a5056..e7c5e548414d54e8ebcaac25aa2677e4819d3195 100644 --- a/src/client/inc/tscParseLine.h +++ b/src/client/inc/tscParseLine.h @@ -28,7 +28,7 @@ typedef struct { } TAOS_SML_KV; typedef struct { - char* stableName; + char* stbName; char* childTableName; TAOS_SML_KV* tags; diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 7338cccfeee184e7f6834e41064e0c71fe5145a0..71efe33011080ee03d3072f68a50047bfebe231f 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -26,7 +26,7 @@ typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg typedef struct { int32_t vgId; char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; // size must same with SVnodeObj.db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] FCqWrite cqWrite; } SCqCfg; @@ -37,7 +37,7 @@ typedef struct { int32_t master; int32_t num; // number of continuous streams char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; struct SCqObj *pHead; diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index ec44a85d5b29c0471db64b0362126804ae73adec..974103052eb190b8f63b15b94482cb9ba4dda455 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17515,7 +17515,7 @@ fun:gaih_inet.constprop.0 fun:getaddrinfo fun:taosGetFqdn - fun:taosCheckGlobalCfg + fun:taosCheckAndPrintCfg fun:taos_init_imp } { @@ -17740,7 +17740,7 @@ fun:gaih_inet.constprop.7 fun:getaddrinfo fun:taosGetFqdn - fun:taosCheckGlobalCfg + fun:taosCheckAndPrintCfg fun:taos_init_imp } {