diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index caf872689c2117c74efcf2c61782a8291a9fa8b8..1fe930ba7abcdaa1b98910048135592542e1ecdf 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -168,8 +168,8 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_SCORES, TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_VNODES, - TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_CLUSTER, + TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_FUNCTION, TSDB_MGMT_TABLE_MAX, @@ -396,17 +396,20 @@ typedef struct { int32_t maxDbs; int32_t maxTimeSeries; int32_t maxStreams; - int64_t maxStorage; // In unit of GB int32_t accessState; // Configured only by command + int64_t maxStorage; // In unit of GB + int32_t reserve[8]; } SCreateAcctMsg, SAlterAcctMsg; typedef struct { - char user[TSDB_USER_LEN]; + char user[TSDB_USER_LEN]; + int32_t reserve[8]; } SDropUserMsg, SDropAcctMsg; typedef struct { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + int32_t reserve[8]; } SCreateUserMsg, SAlterUserMsg; typedef struct { @@ -629,7 +632,6 @@ typedef struct { int32_t reserve[8]; } SUseDbMsg; - typedef struct { char db[TSDB_TABLE_FNAME_LEN]; int32_t reserve[8]; @@ -839,7 +841,7 @@ typedef struct { int32_t tversion; uint64_t tuid; uint64_t suid; - SVgroupMsg vgroup; + int32_t vgId; SSchema pSchema[]; } STableMetaMsg; @@ -867,6 +869,7 @@ typedef struct { int32_t dbVgroupVersion; int32_t dbVgroupNum; int32_t dbHashRange; + int32_t dbHashType; SVgroupInfo vgroupInfo[]; //int32_t vgIdList[]; } SUseDbRspMsg; @@ -897,16 +900,19 @@ typedef struct SShowRsp { } SShowRsp; typedef struct { - char ep[TSDB_EP_LEN]; // end point, hostname:port + char ep[TSDB_EP_LEN]; // end point, hostname:port + int32_t reserve[8]; } SCreateDnodeMsg; typedef struct { int32_t dnodeId; + int32_t reserve[8]; } SDropDnodeMsg; typedef struct { int32_t dnodeId; char config[TSDB_DNODE_CONFIG_LEN]; + int32_t reserve[8]; } SCfgDnodeMsg; typedef struct { diff --git a/include/common/tglobal.h b/include/common/tglobal.h index a9b0e607614811281a63f62349b49b91ed15cb5d..42956b6bdde2b27f3921817b90a1cc871ca0a8dd 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -29,12 +29,7 @@ extern char tsLocalFqdn[]; extern char tsLocalEp[]; extern uint16_t tsServerPort; extern int32_t tsStatusInterval; -extern int32_t tsNumOfMnodes; -extern int8_t tsEnableVnodeBak; extern int8_t tsEnableTelemetryReporting; -extern char tsArbitrator[]; -extern int8_t tsArbOnline; -extern int64_t tsArbOnlineTimestamp; // common extern int tsRpcTimer; @@ -60,7 +55,6 @@ extern int tsCompatibleModel; // 2.0 compatible model extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked - extern int8_t tsKeepOriginalColumnName; // client @@ -78,66 +72,15 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -// db parameters in client -extern int32_t tsCacheBlockSize; -extern int32_t tsBlocksPerVnode; -extern int32_t tsTableIncStepPerVnode; -extern int32_t tsMaxVgroupsPerDb; -extern int16_t tsDaysPerFile; -extern int32_t tsDaysToKeep; -extern int32_t tsMinRowsInFileBlock; -extern int32_t tsMaxRowsInFileBlock; -extern int16_t tsCommitTime; // seconds -extern int32_t tsTimePrecision; -extern int8_t tsCompression; -extern int8_t tsWAL; -extern int32_t tsFsyncPeriod; -extern int32_t tsReplications; -extern int16_t tsPartitons; -extern int32_t tsQuorum; -extern int8_t tsUpdate; -extern int8_t tsCacheLastRow; - -//tsdb -extern bool tsdbForceKeepFile; - // balance -extern int8_t tsEnableBalance; -extern int8_t tsAlternativeRole; -extern int32_t tsBalanceInterval; -extern int32_t tsOfflineThreshold; -extern int8_t tsEnableFlowCtrl; extern int8_t tsEnableSlaveQuery; -extern int8_t tsEnableAdjustMaster; - -// restful -extern int32_t tsRestRowLimit; -extern int8_t tsTelegrafUseFieldNum; - -// mqtt -extern int8_t tsEnableMqttModule; -extern char tsMqttHostName[]; -extern char tsMqttPort[]; -extern char tsMqttUser[]; -extern char tsMqttPass[]; -extern char tsMqttClientId[]; -extern char tsMqttTopic[]; - -// monitor -extern int8_t tsEnableMonitorModule; -extern char tsMonitorDbName[]; -extern char tsInternalPass[]; -extern int32_t tsMonitorInterval; - -// stream -extern int8_t tsEnableStream; - -// internal + + +// interna extern int8_t tsPrintAuth; extern char tsVnodeDir[]; extern char tsMnodeDir[]; extern int64_t tsTickPerDay[3]; -extern int32_t tsTopicBianryLen; // system info extern float tsTotalLogDirGB; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 9620c4ea6231050e83f0cba2cc101cbfc761b304..51d7ca72025db474b34f938d3eb7c90c8d4b023b 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -99,7 +99,7 @@ extern "C" { sdbFreeRaw(pRaw); \ return NULL; \ } \ - dataPos += sizeof(int16_t); \ + dataPos += sizeof(int16_t); \ } #define SDB_SET_INT8(pRaw, dataPos, val) \ @@ -295,6 +295,15 @@ void sdbCancelFetch(SSdb *pSdb, void *pIter); */ int32_t sdbGetSize(SSdb *pSdb, ESdbType type); +/** + * @brief Get the max id of the table, keyType of table should be INT32 + * + * @param pSdb The sdb object. + * @param pIter The type of the table. + * @record int32_t The max id of the table + */ +int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type); + SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); void sdbFreeRaw(SSdbRaw *pRaw); int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val); diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 7359df92cc680c7c052131e843ed158dcf4ff8f4..747d97b7a08992389092f2b9a7b6d69a4e00a208 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -226,21 +226,21 @@ typedef struct TqMetaHandle { int64_t serializedSize; void* valueInUse; void* valueInTxn; -} TqMetaHandle; +} STqMetaHandle; typedef struct TqMetaList { - TqMetaHandle handle; + STqMetaHandle handle; struct TqMetaList* next; //struct TqMetaList* inTxnPrev; //struct TqMetaList* inTxnNext; struct TqMetaList* unpersistPrev; struct TqMetaList* unpersistNext; -} TqMetaList; +} STqMetaList; typedef struct TqMetaStore { - TqMetaList* bucket[TQ_BUCKET_SIZE]; + STqMetaList* bucket[TQ_BUCKET_SIZE]; //a table head - TqMetaList* unpersistHead; + STqMetaList* unpersistHead; //TODO:temporaral use, to be replaced by unified tfile int fileFd; //TODO:temporaral use, to be replaced by unified tfile @@ -250,7 +250,7 @@ typedef struct TqMetaStore { TqSerializeFun pSerializer; TqDeserializeFun pDeserializer; TqDeleteFun pDeleter; -} TqMetaStore; +} STqMetaStore; typedef struct STQ { // the collection of group handle @@ -259,7 +259,7 @@ typedef struct STQ { STqCfg* tqConfig; TqLogReader* tqLogReader; TqMemRef tqMemRef; - TqMetaStore* tqMeta; + STqMetaStore* tqMeta; } STQ; // open in each vnode diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index 26efce5d43f87589888ce00e7d5f378f53b5af0c..f8eac9768fc8ad98d9d7f80095b265fcca5f7e0c 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -21,15 +21,14 @@ extern "C" { #endif // TYPES EXPOSED -typedef struct STsdb STsdb; -typedef struct STsdbCfg STsdbCfg; -typedef struct STsdbMemAllocator STsdbMemAllocator; +typedef struct STsdb STsdb; +typedef struct STsdbCfg STsdbCfg; // STsdb -STsdb *tsdbOpen(const char *path, const STsdbCfg *); +STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg); void tsdbClose(STsdb *); void tsdbRemove(const char *path); -int tsdbInsertData(STsdb *pTsdb, void *pData, int len); +int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg); // STsdbCfg int tsdbOptionsInit(STsdbCfg *); @@ -41,7 +40,6 @@ struct STsdbCfg { uint32_t keep0; uint32_t keep1; uint32_t keep2; - /* TODO */ }; #ifdef __cplusplus diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index f9d3b3c8c18c627df6decfbca87e969aa759fe27..6bbc4f9109c56485771f7b4cdb02380470cb325c 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -32,7 +32,7 @@ extern "C" { struct SCatalog; typedef struct SCatalogReq { - char clusterId[TSDB_CLUSTER_ID_LEN]; //???? + char dbName[TSDB_DB_NAME_LEN]; SArray *pTableName; // table full name SArray *pUdf; // udf name bool qNodeRequired; // valid qnode @@ -45,42 +45,10 @@ typedef struct SMetaData { SEpSet *pEpSet; // qnode epset list } SMetaData; -typedef struct STableComInfo { - uint8_t numOfTags; // the number of tags in schema - uint8_t precision; // the number of precision - int16_t numOfColumns; // the number of columns - int32_t rowSize; // row size of the schema -} STableComInfo; - -/* - * ASSERT(sizeof(SCTableMeta) == 24) - * ASSERT(tableType == TSDB_CHILD_TABLE) - * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info. - */ -typedef struct SCTableMeta { - int32_t vgId:24; - int8_t tableType; - uint64_t uid; - uint64_t suid; -} SCTableMeta; - -/* - * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta. - */ -typedef struct STableMeta { - int32_t vgId:24; - int8_t tableType; - uint64_t uid; - uint64_t suid; - // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info - int16_t sversion; - int16_t tversion; - STableComInfo tableInfo; - SSchema schema[]; -} STableMeta; - typedef struct SCatalogCfg { - + bool enableVgroupCache; + uint32_t maxTblCacheNum; + uint32_t maxDBCacheNum; } SCatalogCfg; int32_t catalogInit(SCatalogCfg *cfg); @@ -96,19 +64,25 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); -int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList); -int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); + +/** + * get cluster vgroup list. + * @pVgroupList - hash of vgroup list, key:vgId, value:SVgroupInfo + * @return + */ +int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash); +int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo); -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); +int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta); -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta); -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, STableMeta* pNewTableMeta); +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); /** @@ -117,7 +91,7 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const * @pVgroupList - array of SVgroupInfo * @return */ -int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, SArray* pVgroupList); +int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList); /** @@ -129,7 +103,7 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe * @param pMetaData * @return */ -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); diff --git a/include/libs/query/query.h b/include/libs/query/query.h index 02ae70887484428710cd224ae389b0de8b18c9bf..bfe2db6a611a6aaa71328136fc163746347fdc31 100644 --- a/include/libs/query/query.h +++ b/include/libs/query/query.h @@ -24,10 +24,49 @@ extern "C" { typedef SVgroupListRspMsg SVgroupListInfo; +typedef struct STableComInfo { + uint8_t numOfTags; // the number of tags in schema + uint8_t precision; // the number of precision + int16_t numOfColumns; // the number of columns + int32_t rowSize; // row size of the schema +} STableComInfo; + +/* + * ASSERT(sizeof(SCTableMeta) == 24) + * ASSERT(tableType == TSDB_CHILD_TABLE) + * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info. + */ +typedef struct SCTableMeta { + int32_t vgId:24; + int8_t tableType; + uint64_t uid; + uint64_t suid; +} SCTableMeta; + +/* + * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta. + */ +typedef struct STableMeta { + //BEGIN: KEEP THIS PART SAME WITH SCTableMeta + int32_t vgId:24; + int8_t tableType; + uint64_t uid; + uint64_t suid; + //END: KEEP THIS PART SAME WITH SCTableMeta + + // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info + int16_t sversion; + int16_t tversion; + STableComInfo tableInfo; + SSchema schema[]; +} STableMeta; + + typedef struct SDBVgroupInfo { int32_t vgroupVersion; SArray *vgId; int32_t hashRange; + int32_t hashType; } SDBVgroupInfo; typedef struct SUseDbOutput { @@ -36,6 +75,13 @@ typedef struct SUseDbOutput { SDBVgroupInfo *dbVgroup; } SUseDbOutput; +typedef struct STableMetaOutput { + int32_t metaNum; + char ctbFname[TSDB_TABLE_FNAME_LEN]; + char tbFname[TSDB_TABLE_FNAME_LEN]; + SCTableMeta ctbMeta; + STableMeta *tbMeta; +} STableMetaOutput; extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index e19d65837a822fd7e8f484944aa6675f97beb7ba..bafdc6c082d1ad389037e102895e99918f06d743 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -88,17 +88,17 @@ typedef struct SWalVer { typedef struct SWal { // cfg SWalCfg cfg; + int32_t fsyncSeq; + //meta SWalVer vers; - //file set int64_t writeLogTfd; int64_t writeIdxTfd; int32_t writeCur; SArray* fileInfoSet; - //statistics + //status int64_t totSize; int64_t lastRollSeq; //ctl - int32_t fsyncSeq; int64_t refId; pthread_mutex_t mutex; //path diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8400031f9bdfcf0979725f85261d93426d4d502c..4f1ef7da7b6fa0a93fdb2d3cb20bf9f45c57ae91 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -119,71 +119,91 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") #define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input") -// mnode -#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) -#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) -#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302) -#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0304) -#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) -#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0306) -#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0307) -#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0308) -#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x0309) -#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x030B) -#define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x030C) -#define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D) -#define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x030E) -#define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0310) -#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0311) -#define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0312) -#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0313) -#define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) - -#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0320) -#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0321) -#define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0322) -#define TSDB_CODE_SDB_OBJ_CREATING TAOS_DEF_ERROR_CODE(0, 0x0323) -#define TSDB_CODE_SDB_OBJ_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0324) -#define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) -#define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0326) -#define TSDB_CODE_SDB_INVALID_ACTION_TYPE TAOS_DEF_ERROR_CODE(0, 0x0327) -#define TSDB_CODE_SDB_INVALID_STATUS_TYPE TAOS_DEF_ERROR_CODE(0, 0x0328) -#define TSDB_CODE_SDB_INVALID_DATA_VER TAOS_DEF_ERROR_CODE(0, 0x0329) -#define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x032A) -#define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x032B) +// mnode-common +#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0300) +#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0301) +#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302) +#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0303) +#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0304) +#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0305) +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0306) +#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0307) +#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0308) +#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0309) +#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030A) + +// mnode-show +#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310) + +// mnode-profile +#define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x0320) +#define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x0321) +#define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x0322) +#define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0323) +#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0324) +#define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0325) +#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0326) +#define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0327) + +// mnode-sdb +#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0330) +#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0331) +#define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0332) +#define TSDB_CODE_SDB_OBJ_CREATING TAOS_DEF_ERROR_CODE(0, 0x0333) +#define TSDB_CODE_SDB_OBJ_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0334) +#define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0335) +#define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0336) +#define TSDB_CODE_SDB_INVALID_ACTION_TYPE TAOS_DEF_ERROR_CODE(0, 0x0337) +#define TSDB_CODE_SDB_INVALID_STATUS_TYPE TAOS_DEF_ERROR_CODE(0, 0x0338) +#define TSDB_CODE_SDB_INVALID_DATA_VER TAOS_DEF_ERROR_CODE(0, 0x0339) +#define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x033A) +#define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x033B) // mnode-dnode -#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) -#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) -#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0332) -#define TSDB_CODE_MND_INVALID_CLUSTER_CFG TAOS_DEF_ERROR_CODE(0, 0x0333) -#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x0334) -#define TSDB_CODE_MND_INVALID_DNODE_CFG TAOS_DEF_ERROR_CODE(0, 0x0335) -#define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0336) -#define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0337) - -// mnode-vgroup -#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0332) //"VGroup does not exist") -#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0338) //"Vgroup not in dnode") -#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0339) //"Vgroup already in dnode") -#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x033C) //"Cluster not ready") +#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) +#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0341) +#define TSDB_CODE_MND_TOO_MANY_DNODES TAOS_DEF_ERROR_CODE(0, 0x0342) +#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0343) +#define TSDB_CODE_MND_INVALID_CLUSTER_CFG TAOS_DEF_ERROR_CODE(0, 0x0344) +#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x0345) +#define TSDB_CODE_MND_INVALID_DNODE_CFG TAOS_DEF_ERROR_CODE(0, 0x0346) +#define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0347) +#define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0348) + +// mnode-mnode +#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) +#define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351) +#define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x0352) // mnode-acct -#define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) //"Account already exists") -#define TSDB_CODE_MND_ACCT_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0341) //"Invalid account") -#define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0342) //"Invalid account options") -#define TSDB_CODE_MND_ACCT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0343) //"Account authorization has expired") - -#define TSDB_CODE_MND_USER_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) //"User already exists") -#define TSDB_CODE_MND_USER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351) //"Invalid user") -#define TSDB_CODE_MND_INVALID_USER_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0352) //"Invalid user format") -#define TSDB_CODE_MND_INVALID_PASS_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0353) //"Invalid password format") -#define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0354) //"Can not get user from conn") -#define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0355) //"Too many users") +#define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_ACCT_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0361) +#define TSDB_CODE_MND_TOO_MANY_ACCTS TAOS_DEF_ERROR_CODE(0, 0x0362) +#define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0363) +#define TSDB_CODE_MND_ACCT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0364) + +// mnode-user +#define TSDB_CODE_MND_USER_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0370) +#define TSDB_CODE_MND_USER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0371) +#define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0372) +#define TSDB_CODE_MND_INVALID_USER_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0373) +#define TSDB_CODE_MND_INVALID_PASS_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0374) +#define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0375) + +// mnode-db +#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0380) +#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) +#define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0382) +#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0384) +#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0385) +#define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0386) +#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0387) -#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0348) //"Mnode already exists") -#define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0349) //"Mnode not there") +// mnode-vgroup +#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0390) +#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0391) +#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0392) // mnode-stable #define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) @@ -224,30 +244,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") -#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) -#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) -#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) -#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) -#define TSDB_CODE_MND_INVALID_DB_CACHE_SIZE TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_DAYS TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_KEEP0 TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_KEEP1 TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_KEEP2 TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_MIN_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_MAX_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_COMMIT_TIME TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_PRECISION TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_COMP TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_QUORUM TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_UPDATE TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_CACHE_LAST TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) //"Too many databases for account") // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/include/util/tdef.h b/include/util/tdef.h index 300b190e97a9093298ae849664f688975fcec7fe..85b51b6eff18cb543561c04637fe186595225d83 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -25,7 +25,9 @@ extern "C" { #define TSDB__packed #define TSKEY int64_t -#define TSKEY_INITIAL_VAL INT64_MIN +#define TSKEY_MIN INT64_MIN +#define TSKEY_MAX (INT64_MAX - 1) +#define TSKEY_INITIAL_VAL TSKEY_MIN // Bytes for each type. extern const int32_t TYPE_BYTES[15]; @@ -209,6 +211,7 @@ do { \ #define TSDB_STEP_NAME_LEN 32 #define TSDB_STEP_DESC_LEN 128 +#define TSDB_ERROR_MSG_LEN 1024 #define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_MQTT_HOSTNAME_LEN 64 diff --git a/include/util/tlog.h b/include/util/tlog.h index 9b6033e7fe550bff0c34dadf42e8d139aa703fe9..5e6604598dd22ad7755e4af51921fa3ee80a6abe 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -32,7 +32,6 @@ extern int32_t mDebugFlag; extern int32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; -extern int32_t sdbDebugFlag; extern int32_t httpDebugFlag; extern int32_t mqttDebugFlag; extern int32_t monDebugFlag; diff --git a/source/common/inc/commonInt.h b/source/common/inc/commonInt.h index e7d2dba95c7084ab061b82e9c2e81adb46953aad..5b71f83faf477f217bc2183dc258edccf76913f7 100644 --- a/source/common/inc/commonInt.h +++ b/source/common/inc/commonInt.h @@ -20,6 +20,7 @@ extern "C" { #endif +extern bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index dfbbe33b59850ccc1bb29dffbbcaf4027c434a8b..910d05e9a3eec9518f76891240cd8f5af22e4c6d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -30,19 +30,12 @@ // cluster char tsFirst[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0}; -char tsArbitrator[TSDB_EP_LEN] = {0}; char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port uint16_t tsServerPort = 6030; int32_t tsStatusInterval = 1; // second -int32_t tsNumOfMnodes = 1; -int8_t tsEnableVnodeBak = 1; int8_t tsEnableTelemetryReporting = 0; -int8_t tsArbOnline = 0; -int64_t tsArbOnlineTimestamp = TSDB_ARB_DUMMY_TIME; char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsDnodeId = 0; -int64_t tsDnodeStartTime = 0; // common int32_t tsRpcTimer = 300; @@ -129,59 +122,19 @@ int32_t tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name int8_t tsKeepOriginalColumnName = 0; -// db parameters -int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; -int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS; -int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; -int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP; -int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; -int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; -int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds -int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; -int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; -int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; -int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; -int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; -int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; -int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; -int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; -int32_t tsMaxVgroupsPerDb = 0; -int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO; - // tsdb config // For backward compatibility bool tsdbForceKeepFile = false; // balance -int8_t tsEnableBalance = 1; -int8_t tsAlternativeRole = 0; -int32_t tsBalanceInterval = 300; // seconds -int32_t tsOfflineThreshold = 86400 * 10; // seconds of 10 days int8_t tsEnableFlowCtrl = 1; int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; -// restful -int32_t tsRestRowLimit = 10240; -int8_t tsTelegrafUseFieldNum = 0; - -// mqtt -int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default -char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org"; -char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883"; -char tsMqttUser[TSDB_MQTT_USER_LEN] = {0}; -char tsMqttPass[TSDB_MQTT_PASS_LEN] = {0}; -char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber"; -char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // # // monitor -int8_t tsEnableMonitorModule = 1; char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; char tsInternalPass[] = "secretkey"; -int32_t tsMonitorInterval = 30; // seconds - -// stream -int8_t tsEnableStream = 1; // internal int8_t tsCompactMnodeWal = 0; @@ -191,7 +144,6 @@ char tsDnodeDir[PATH_MAX] = {0}; char tsMnodeDir[PATH_MAX] = {0}; int32_t tsDiskCfgNum = 0; -int32_t tsTopicBianryLen = 16000; #ifndef _STORAGE SDiskCfg tsDiskCfg[1]; @@ -245,14 +197,10 @@ static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; void taosSetAllDebugFlag() { if (debugFlag != 0) { mDebugFlag = debugFlag; - sdbDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; jniDebugFlag = debugFlag; odbcDebugFlag = debugFlag; - httpDebugFlag = debugFlag; - mqttDebugFlag = debugFlag; - monDebugFlag = debugFlag; qDebugFlag = debugFlag; rpcDebugFlag = debugFlag; uDebugFlag = debugFlag; @@ -459,16 +407,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "arbitrator"; - cfg.ptr = tsArbitrator; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - // dnode configs cfg.option = "numOfThreadsPerCore"; cfg.ptr = &tsNumOfThreadsPerCore; @@ -509,26 +447,6 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - - cfg.option = "numOfMnodes"; - cfg.ptr = &tsNumOfMnodes; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 3; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "vnodeBak"; - cfg.ptr = &tsEnableVnodeBak; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; @@ -540,37 +458,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "balance"; - cfg.ptr = &tsEnableBalance; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "balanceInterval"; - cfg.ptr = &tsBalanceInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 30000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // 0-any; 1-mnode; 2-vnode - cfg.option = "role"; - cfg.ptr = &tsAlternativeRole; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 2; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - // timer cfg.option = "maxTmrCtrl"; cfg.ptr = &tsMaxTmrCtrl; @@ -582,26 +469,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "monitorInterval"; - cfg.ptr = &tsMonitorInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 600; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "offlineThreshold"; - cfg.ptr = &tsOfflineThreshold; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 3; - cfg.maxValue = 86400 * 365; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - cfg.option = "rpcTimer"; cfg.ptr = &tsRpcTimer; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -712,186 +579,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "maxVgroupsPerDb"; - cfg.ptr = &tsMaxVgroupsPerDb; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 8192; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cache"; - cfg.ptr = &tsCacheBlockSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE; - cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MB; - taosAddConfigOption(cfg); - - cfg.option = "blocks"; - cfg.ptr = &tsBlocksPerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TOTAL_BLOCKS; - cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "days"; - cfg.ptr = &tsDaysPerFile; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DAYS_PER_FILE; - cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "keep"; - cfg.ptr = &tsDaysToKeep; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_KEEP; - cfg.maxValue = TSDB_MAX_KEEP; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "minRows"; - cfg.ptr = &tsMinRowsInFileBlock; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK; - cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxRows"; - cfg.ptr = &tsMaxRowsInFileBlock; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK; - cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "precision"; - cfg.ptr = &tsTimePrecision; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_PRECISION; - cfg.maxValue = TSDB_MAX_PRECISION; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "comp"; - cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_COMP_LEVEL; - cfg.maxValue = TSDB_MAX_COMP_LEVEL; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "walLevel"; - cfg.ptr = &tsWAL; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_WAL_LEVEL; - cfg.maxValue = TSDB_MAX_WAL_LEVEL; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fsync"; - cfg.ptr = &tsFsyncPeriod; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_FSYNC_PERIOD; - cfg.maxValue = TSDB_MAX_FSYNC_PERIOD; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "replica"; - cfg.ptr = &tsReplications; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION; - cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "quorum"; - cfg.ptr = &tsQuorum; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_QUORUM_OPTION; - cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "update"; - cfg.ptr = &tsUpdate; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_UPDATE; - cfg.maxValue = TSDB_MAX_DB_UPDATE; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cachelast"; - cfg.ptr = &tsCacheLastRow; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_CACHE_LAST_ROW; - cfg.maxValue = TSDB_MAX_DB_CACHE_LAST_ROW; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqttHostName"; - cfg.ptr = tsMqttHostName; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_MQTT_HOSTNAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqttPort"; - cfg.ptr = tsMqttPort; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_MQTT_PORT_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqttTopic"; - cfg.ptr = tsMqttTopic; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_MQTT_TOPIC_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "compressMsgSize"; cfg.ptr = &tsCompressMsgSize; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1085,76 +772,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "adjustMaster"; - cfg.ptr = &tsEnableAdjustMaster; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqtt"; - cfg.ptr = &tsEnableMqttModule; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "monitor"; - cfg.ptr = &tsEnableMonitorModule; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "stream"; - cfg.ptr = &tsEnableStream; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "topicBianryLen"; - cfg.ptr = &tsTopicBianryLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 16; - cfg.maxValue = 16000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "telegrafUseFieldNum"; - cfg.ptr = &tsTelegrafUseFieldNum; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "restfulRowLimit"; - cfg.ptr = &tsRestRowLimit; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - // debug flag cfg.option = "numOfLogLines"; cfg.ptr = &tsNumOfLogLines; @@ -1236,17 +853,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - - cfg.option = "sdbDebugFlag"; - cfg.ptr = &sdbDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "rpcDebugFlag"; cfg.ptr = &rpcDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1307,36 +913,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "httpDebugFlag"; - cfg.ptr = &httpDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mqttDebugFlag"; - cfg.ptr = &mqttDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "monDebugFlag"; - cfg.ptr = &monDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "qDebugFlag"; cfg.ptr = &qDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1456,16 +1032,6 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = PATH_MAX; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tsdbMetaCompactRatio"; - cfg.ptr = &tsTsdbMetaCompactRatio; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 100; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); // enable kill long query @@ -1630,4 +1196,3 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *d return true; } - \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index b340029044c7b0f23372010872aa06b2b75146a1..86186f77de7d05a635bbca7a2090025a9098dec3 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,6 +1,19 @@ -# add_subdirectory(acct) + +add_subdirectory(acct) +# add_subdirectory(auth) +# add_subdirectory(balance) # add_subdirectory(cluster) +add_subdirectory(db) add_subdirectory(dnode) +# add_subdirectory(func) +# add_subdirectory(mnode) # add_subdirectory(profile) # add_subdirectory(show) +# add_subdirectory(stb) +# add_subdirectory(sync) +# add_subdirectory(telem) +# add_subdirectory(trans) add_subdirectory(user) +# add_subdirectory(vgroup) + +# add_subdirectory(common) diff --git a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt index 3e963df2e652071192e56ac212ef6ab23f351309..a548c2adc2d32a4a525acd1ab9bcbf23dcf4ce05 100644 --- a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTestAcct "") +add_executable(dnode_test_acct "") -target_sources(dndTestAcct +target_sources(dnode_test_acct PRIVATE "acct.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestAcct + dnode_test_acct PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestAcct +target_include_directories(dnode_test_acct PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTestAcct enable_testing() add_test( - NAME dndTestAcct - COMMAND dndTestAcct + NAME dnode_test_acct + COMMAND dnode_test_acct ) diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index 9050a938fa54634d20e7a8981b864c546ca53d9b..e1a71c5c9de71580f2578c1bba7a78c3095b9bdd 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -17,24 +17,27 @@ class DndTestAcct : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestAcct"; - const char* fqdn = "localhost"; - uint16_t port = 9520; + initLog("/tmp/tdlog"); - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9012"; + pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9012, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9012); + taosMsleep(300); } static void TearDownTestSuite() { stopServer(pServer); dropClient(pClient); + pServer = NULL; + pClient = NULL; } static SServer* pServer; diff --git a/source/dnode/mgmt/impl/test/db/CMakeLists.txt b/source/dnode/mgmt/impl/test/db/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b778e3854f56325682b01334ba78ae23b22e0eae --- /dev/null +++ b/source/dnode/mgmt/impl/test/db/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dnode_test_db "") + +target_sources(dnode_test_db + PRIVATE + "db.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dnode_test_db + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dnode_test_db + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dnode_test_db + COMMAND dnode_test_db +) diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b0ee00cb64a52272283d0ff9866309b864fd79b9 --- /dev/null +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestDb : public ::testing::Test { + protected: + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } + + static void SetUpTestSuite() { + initLog("/tmp/tdlog"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9040"; + pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9040); + taosMsleep(300); + } + + static void TearDownTestSuite() { + stopServer(pServer); + dropClient(pClient); + pServer = NULL; + pClient = NULL; + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htons(pMeta->numOfTags); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt8(int8_t val) { + int8_t data = *((int8_t*)(pData + pos)); + pos += sizeof(int8_t); + EXPECT_EQ(data, val); + } + + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); + } + + void CheckInt32(int32_t val) { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + EXPECT_EQ(data, val); + } + + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_EQ(data, val); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(const char* val, int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; + +SServer* DndTestDb::pServer; +SClient* DndTestDb::pClient; +int32_t DndTestDb::connId; + +TEST_F(DndTestDb, 01_ShowDb) { + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); + CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); + CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); + CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "days"); + CheckSchema(5, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); + CheckSchema(6, TSDB_DATA_TYPE_INT, 4, "cache(MB)"); + CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "blocks"); + CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "minrows"); + CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "maxrows"); + CheckSchema(10, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); + CheckSchema(11, TSDB_DATA_TYPE_INT, 4, "fsync"); + CheckSchema(12, TSDB_DATA_TYPE_TINYINT, 1, "comp"); + CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); + CheckSchema(14, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); + CheckSchema(15, TSDB_DATA_TYPE_TINYINT, 1, "update"); + + SendThenCheckShowRetrieveMsg(0); +} + +TEST_F(DndTestDb, 02_CreateDb) { + { + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "1.d1"); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(1); // replica + CheckInt16(1); // quorum + CheckInt16(10); // days + CheckBinary("3650,3650,3650", 24); // days + CheckInt32(16); // cache + CheckInt32(10); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(1); // wallevel + CheckInt32(3000); // fsync + CheckInt8(2); // comp + CheckInt8(0); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update +} + +TEST_F(DndTestDb, 03_AlterDb) { + { + SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + strcpy(pReq->db, "1.d1"); + pReq->totalBlocks = htonl(12); + pReq->daysToKeep0 = htonl(300); + pReq->daysToKeep1 = htonl(400); + pReq->daysToKeep2 = htonl(500); + pReq->fsyncPeriod = htonl(4000); + pReq->walLevel = 2; + pReq->quorum = 2; + pReq->cacheLastRow = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(1); // replica + CheckInt16(2); // quorum + CheckInt16(10); // days + CheckBinary("300,400,500", 24); // days + CheckInt32(16); // cache + CheckInt32(12); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(2); // wallevel + CheckInt32(4000); // fsync + CheckInt8(2); // comp + CheckInt8(1); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update +} + +TEST_F(DndTestDb, 04_RestartDnode) { + stopServer(pServer); + pServer = NULL; + + uInfo("start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9040"; + pServer = startServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); + + uInfo("all server is running"); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(1); // replica + CheckInt16(2); // quorum + CheckInt16(10); // days + CheckBinary("300,400,500", 24); // days + CheckInt32(16); // cache + CheckInt32(12); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(2); // wallevel + CheckInt32(4000); // fsync + CheckInt8(2); // comp + CheckInt8(1); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update +} + +TEST_F(DndTestDb, 05_DropDb) { + { + SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + strcpy(pReq->db, "1.d1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendThenCheckShowRetrieveMsg(0); +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt index e118cb872559eab22e1210d37fee11f30bf281c4..ebe2f3c5eb8b20122349f3f83278dce6f65b8bff 100644 --- a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTestDnode "") +add_executable(dnode_test_dnode "") -target_sources(dndTestDnode +target_sources(dnode_test_dnode PRIVATE "dnode.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestDnode + dnode_test_dnode PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestDnode +target_include_directories(dnode_test_dnode PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTestDnode enable_testing() add_test( - NAME dndTestDnode - COMMAND dndTestDnode + NAME dnode_test_dnode + COMMAND dnode_test_dnode ) diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 580fe8e131643bee660162a3437eba41300a0fc2..5a4512e16c38645accd8a0474f06468f7c58ae2d 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -24,16 +24,16 @@ class DndTestDnode : public ::testing::Test { } static void SetUpTestSuite() { - initLog("/tmp/dndTestDnode"); + initLog("/tmp/tdlog"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9521"; - pServer1 = CreateServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); - pServer2 = CreateServer("/tmp/dndTestDnode2", fqdn, 9522, firstEp); - pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); - pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); - pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9521); + const char* firstEp = "localhost:9041"; + pServer1 = CreateServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp); + pServer2 = CreateServer("/tmp/dnode_test_dnode2", fqdn, 9042, firstEp); + pServer3 = CreateServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); + pServer4 = CreateServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); + pServer5 = CreateServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9041); taosMsleep(300); } @@ -188,7 +188,7 @@ SServer* DndTestDnode::pServer4; SServer* DndTestDnode::pServer5; SClient* DndTestDnode::pClient; -TEST_F(DndTestDnode, ShowDnode) { +TEST_F(DndTestDnode, 01_ShowDnode) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point"); @@ -200,7 +200,7 @@ TEST_F(DndTestDnode, ShowDnode) { SendThenCheckShowRetrieveMsg(1); CheckInt16(1); - CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); CheckInt16(1); CheckBinary("ready", 10); @@ -208,7 +208,7 @@ TEST_F(DndTestDnode, ShowDnode) { CheckBinary("", 24); } -TEST_F(DndTestDnode, ConfigDnode_01) { +TEST_F(DndTestDnode, 02_ConfigDnode) { SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg)); pReq->dnodeId = htonl(1); strcpy(pReq->config, "ddebugflag 131"); @@ -224,9 +224,9 @@ TEST_F(DndTestDnode, ConfigDnode_01) { ASSERT_EQ(pMsg->code, 0); } -TEST_F(DndTestDnode, CreateDnode_01) { +TEST_F(DndTestDnode, 03_CreateDnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9522"); + strcpy(pReq->ep, "localhost:9042"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -243,8 +243,8 @@ TEST_F(DndTestDnode, CreateDnode_01) { SendThenCheckShowRetrieveMsg(2); CheckInt16(1); CheckInt16(2); - CheckBinary("localhost:9521", TSDB_EP_LEN); - CheckBinary("localhost:9522", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); + CheckBinary("localhost:9042", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(1); @@ -257,7 +257,7 @@ TEST_F(DndTestDnode, CreateDnode_01) { CheckBinary("", 24); } -TEST_F(DndTestDnode, DropDnode_01) { +TEST_F(DndTestDnode, 04_DropDnode) { SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); pReq->dnodeId = htonl(2); @@ -274,7 +274,7 @@ TEST_F(DndTestDnode, DropDnode_01) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); SendThenCheckShowRetrieveMsg(1); CheckInt16(1); - CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); CheckInt16(1); CheckBinary("ready", 10); @@ -282,10 +282,10 @@ TEST_F(DndTestDnode, DropDnode_01) { CheckBinary("", 24); } -TEST_F(DndTestDnode, CreateDnode_02) { +TEST_F(DndTestDnode, 05_CreateDnode) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9523"); + strcpy(pReq->ep, "localhost:9043"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -300,7 +300,7 @@ TEST_F(DndTestDnode, CreateDnode_02) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9524"); + strcpy(pReq->ep, "localhost:9044"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -315,7 +315,7 @@ TEST_F(DndTestDnode, CreateDnode_02) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9525"); + strcpy(pReq->ep, "localhost:9045"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -335,10 +335,10 @@ TEST_F(DndTestDnode, CreateDnode_02) { CheckInt16(3); CheckInt16(4); CheckInt16(5); - CheckBinary("localhost:9521", TSDB_EP_LEN); - CheckBinary("localhost:9523", TSDB_EP_LEN); - CheckBinary("localhost:9524", TSDB_EP_LEN); - CheckBinary("localhost:9525", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); + CheckBinary("localhost:9043", TSDB_EP_LEN); + CheckBinary("localhost:9044", TSDB_EP_LEN); + CheckBinary("localhost:9045", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(0); @@ -361,7 +361,7 @@ TEST_F(DndTestDnode, CreateDnode_02) { CheckBinary("", 24); } -TEST_F(DndTestDnode, RestartDnode_01) { +TEST_F(DndTestDnode, 06_RestartDnode) { uInfo("stop all server"); stopServer(pServer1); stopServer(pServer2); @@ -377,11 +377,11 @@ TEST_F(DndTestDnode, RestartDnode_01) { uInfo("start all server"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9521"; - pServer1 = startServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); - pServer3 = startServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); - pServer4 = startServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); - pServer5 = startServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + const char* firstEp = "localhost:9041"; + pServer1 = startServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp); + pServer3 = startServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); + pServer4 = startServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); + pServer5 = startServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); uInfo("all server is running"); @@ -392,10 +392,10 @@ TEST_F(DndTestDnode, RestartDnode_01) { CheckInt16(3); CheckInt16(4); CheckInt16(5); - CheckBinary("localhost:9521", TSDB_EP_LEN); - CheckBinary("localhost:9523", TSDB_EP_LEN); - CheckBinary("localhost:9524", TSDB_EP_LEN); - CheckBinary("localhost:9525", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); + CheckBinary("localhost:9043", TSDB_EP_LEN); + CheckBinary("localhost:9044", TSDB_EP_LEN); + CheckBinary("localhost:9045", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(0); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index be1506bccf7a4f87cfeb6ff400ff0a511dd5bf7f..f2010b581388f24359ff2c15f4517a3fded22047 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -22,10 +22,6 @@ void initLog(const char* path) { cDebugFlag = 0; jniDebugFlag = 0; tmrDebugFlag = 0; - sdbDebugFlag = 0; - httpDebugFlag = 0; - mqttDebugFlag = 0; - monDebugFlag = 0; uDebugFlag = 143; rpcDebugFlag = 0; odbcDebugFlag = 0; diff --git a/source/dnode/mgmt/impl/test/user/CMakeLists.txt b/source/dnode/mgmt/impl/test/user/CMakeLists.txt index b5f02e41f4c594e353ce02b74f1feb42de205683..ca8f2ec6db61700893d2381e5c129cff1b6b5dad 100644 --- a/source/dnode/mgmt/impl/test/user/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/user/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTestUser "") +add_executable(dnode_test_user "") -target_sources(dndTestUser +target_sources(dnode_test_user PRIVATE "user.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestUser + dnode_test_user PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestUser +target_include_directories(dnode_test_user PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTestUser enable_testing() add_test( - NAME dndTestUser - COMMAND dndTestUser + NAME dnode_test_user + COMMAND dnode_test_user ) diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 48be2635cddb2903d58e8d10ee1263bb0355e48c..2b30dcbb1bb4a79db6fe6eac43f35bfa71762905 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -24,12 +24,12 @@ class DndTestUser : public ::testing::Test { } static void SetUpTestSuite() { - initLog("/tmp/dndTestUser"); + initLog("/tmp/tdlog"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9530"; - pServer = CreateServer("/tmp/dndTestUser", fqdn, 9530, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9530); + const char* firstEp = "localhost:9140"; + pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9140, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9140); taosMsleep(300); } @@ -170,7 +170,7 @@ SServer* DndTestUser::pServer; SClient* DndTestUser::pClient; int32_t DndTestUser::connId; -TEST_F(DndTestUser, ShowUser) { +TEST_F(DndTestUser, 01_ShowUser) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); @@ -184,7 +184,7 @@ TEST_F(DndTestUser, ShowUser) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, CreateUser_01) { +TEST_F(DndTestUser, 02_CreateUser) { { SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); strcpy(pReq->user, "u1"); @@ -233,7 +233,7 @@ TEST_F(DndTestUser, CreateUser_01) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, AlterUser_01) { +TEST_F(DndTestUser, 03_AlterUser) { SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p2"); @@ -264,7 +264,7 @@ TEST_F(DndTestUser, AlterUser_01) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, DropUser_01) { +TEST_F(DndTestUser, 04_DropUser) { SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); strcpy(pReq->user, "u1"); @@ -290,15 +290,15 @@ TEST_F(DndTestUser, DropUser_01) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, RestartDnode) { +TEST_F(DndTestUser, 05_RestartDnode) { stopServer(pServer); pServer = NULL; uInfo("start all server"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9530"; - pServer = startServer("/tmp/dndTestUser", fqdn, 9530, firstEp); + const char* firstEp = "localhost:9140"; + pServer = startServer("/tmp/dnode_test_user", fqdn, 9140, firstEp); uInfo("all server is running"); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 0f4839392e66212c951237582f8c035398694456..115d896da25f30482a0430c6bc8058021ac58e75 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -17,14 +17,16 @@ #define _TD_MND_DEF_H_ #include "os.h" + +#include "cJSON.h" +#include "sync.h" #include "taosmsg.h" +#include "thash.h" #include "tlog.h" #include "trpc.h" #include "ttimer.h" -#include "thash.h" -#include "cJSON.h" + #include "mnode.h" -#include "sync.h" #ifdef __cplusplus extern "C" { @@ -41,11 +43,8 @@ extern int32_t mDebugFlag; #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} typedef struct SClusterObj SClusterObj; -typedef struct SDnodeObj SDnodeObj; typedef struct SMnodeObj SMnodeObj; typedef struct SAcctObj SAcctObj; -typedef struct SUserObj SUserObj; -typedef struct SDbObj SDbObj; typedef struct SVgObj SVgObj; typedef struct SFuncObj SFuncObj; typedef struct SOperObj SOperObj; @@ -119,7 +118,7 @@ typedef struct SClusterObj { int64_t updateTime; } SClusterObj; -typedef struct SDnodeObj { +typedef struct { int32_t id; int64_t createdTime; int64_t updateTime; @@ -178,7 +177,7 @@ typedef struct SAcctObj { SAcctInfo info; } SAcctObj; -typedef struct SUserObj { +typedef struct { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char acct[TSDB_USER_LEN]; @@ -209,12 +208,13 @@ typedef struct { int8_t cacheLastRow; } SDbCfg; -typedef struct SDbObj { +typedef struct { char name[TSDB_FULL_DB_NAME_LEN]; char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; int64_t uid; + int32_t version; SDbCfg cfg; } SDbObj; diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index 37fb9159dc3f18ab372c2d0d65aac336f1c73158..67e277677c91c5bf7eeab7a82cebe020acde2754 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -27,7 +27,7 @@ void mndCleanupShow(SMnode *pMnode); void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); -void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); +void mndVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); char *mndShowStr(int32_t showType); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index a365ad7326ba2367f57364b81d1031dc6e9a80cb..75b0acbd0a6df8e98b1a3304edb15701284805dc 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -19,12 +19,12 @@ #define SDB_ACCT_VER 1 -static int32_t mnodeCreateDefaultAcct(SMnode *pMnode); -static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct); -static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw); -static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); -static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); -static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct); +static int32_t mndCreateDefaultAcct(SMnode *pMnode); +static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct); +static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw); +static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); +static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); +static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct); static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); @@ -32,12 +32,12 @@ static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); int32_t mndInitAcct(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_ACCT, .keyType = SDB_KEY_BINARY, - .deployFp = mnodeCreateDefaultAcct, - .encodeFp = (SdbEncodeFp)mnodeAcctActionEncode, - .decodeFp = (SdbDecodeFp)mnodeAcctActionDecode, - .insertFp = (SdbInsertFp)mnodeAcctActionInsert, - .updateFp = (SdbUpdateFp)mnodeAcctActionUpdate, - .deleteFp = (SdbDeleteFp)mnodeAcctActionDelete}; + .deployFp = mndCreateDefaultAcct, + .encodeFp = (SdbEncodeFp)mndAcctActionEncode, + .decodeFp = (SdbDecodeFp)mndAcctActionDecode, + .insertFp = (SdbInsertFp)mndAcctActionInsert, + .updateFp = (SdbUpdateFp)mndAcctActionUpdate, + .deleteFp = (SdbDeleteFp)mndAcctActionDelete}; mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_ACCT, mndProcessCreateAcctMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_ACCT, mndProcessAlterAcctMsg); @@ -48,7 +48,7 @@ int32_t mndInitAcct(SMnode *pMnode) { void mndCleanupAcct(SMnode *pMnode) {} -static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { +static int32_t mndCreateDefaultAcct(SMnode *pMnode) { SAcctObj acctObj = {0}; tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); acctObj.createdTime = taosGetTimestampMs(); @@ -61,7 +61,7 @@ static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { .maxStorage = INT64_MAX, .accessState = TSDB_VN_ALL_ACCCESS}; - SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj); + SSdbRaw *pRaw = mndAcctActionEncode(&acctObj); if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); @@ -69,7 +69,7 @@ static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } -static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) { +static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); if (pRaw == NULL) return NULL; @@ -90,7 +90,7 @@ static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) { return pRaw; } -static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw) { +static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; @@ -120,18 +120,18 @@ static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw) { return pRow; } -static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { +static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { mTrace("acct:%s, perform insert action", pAcct->acct); memset(&pAcct->info, 0, sizeof(SAcctInfo)); return 0; } -static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { +static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { mTrace("acct:%s, perform delete action", pAcct->acct); return 0; } -static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { +static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { mTrace("acct:%s, perform update action", pOldAcct->acct); memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 7edeb736470faead055430ee02f3b1d54acafddf..0eaa1849077580bde673605fbf485189aebf98f8 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -203,7 +203,7 @@ static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, numOfRows++; } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b638728647c277761febf3869dff438fb75d5c8f..9f2b2d17f37a708c3c948f8265033870520670e1 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -65,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) { void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -74,6 +74,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT64(pRaw, dataPos, pDb->createdTime) SDB_SET_INT64(pRaw, dataPos, pDb->updateTime) SDB_SET_INT64(pRaw, dataPos, pDb->uid) + SDB_SET_INT32(pRaw, dataPos, pDb->version) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) @@ -117,6 +118,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->version) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) @@ -151,7 +153,8 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { mTrace("db:%s, perform update action", pOldDb->name); - memcpy(pOldDb, pNewDb, sizeof(SDbObj)); + pOldDb->updateTime = pNewDb->createdTime; + memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); return 0; } @@ -165,109 +168,146 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { +static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) { + char *pos = strstr(dbName, TS_PATH_DELIMITER); + if (pos == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + return -1; + } + + int32_t acctId = atoi(dbName); + if (acctId != pUser->acctId) { + terrno = TSDB_CODE_MND_INVALID_DB_ACCT; + return -1; + } + + return 0; +} + +static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg, char *errMsg, int32_t len) { if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - terrno = TSDB_CODE_MND_INVALID_DB_CACHE_SIZE; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database cache block size option", len); return -1; } if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { - terrno = TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database total blocks option", len); return -1; } if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { - terrno = TSDB_CODE_MND_INVALID_DB_DAYS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database days option", len); return -1; } if (pCfg->daysToKeep0 < pCfg->daysPerFile) { - terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database days option", len); return -1; } if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) { - terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database keep0 option", len); return -1; } if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) { - terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database keep1 option", len); return -1; } if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) { - terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database keep2 option", len); return -1; } if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { - terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database minrows option", len); return -1; } if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { - terrno = TSDB_CODE_MND_INVALID_DB_MAX_ROWS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database maxrows option", len); return -1; } if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { - terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database minrows option", len); return -1; } if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { - terrno = TSDB_CODE_MND_INVALID_DB_COMMIT_TIME; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database commit option", len); return -1; } if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) { - terrno = TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database fsync option", len); return -1; } if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) { - terrno = TSDB_CODE_MND_INVALID_DB_WAL_LEVEL; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database wal level option", len); return -1; } if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { - terrno = TSDB_CODE_MND_INVALID_DB_PRECISION; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid precision option", len); return -1; } if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { - terrno = TSDB_CODE_MND_INVALID_DB_COMP; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database compression option", len); return -1; } if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) { - terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database replication option", len); return -1; } if (pCfg->replications > mndGetDnodeSize(pMnode)) { - terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database replication option", len); return -1; } if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) { - terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database quorum option", len); return -1; } if (pCfg->quorum > pCfg->replications) { - terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database quorum option", len); return -1; } if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { - terrno = TSDB_CODE_MND_INVALID_DB_UPDATE; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database update option", len); return -1; } if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { - terrno = TSDB_CODE_MND_INVALID_DB_CACHE_LAST; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database cachelast option", len); return -1; } @@ -294,13 +334,13 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; } -static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, char *acct) { +static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); - tstrncpy(dbObj.acct, acct, TSDB_USER_LEN); + tstrncpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; - dbObj.uid = 1234; + dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize, .totalBlocks = pCreate->totalBlocks, .daysPerFile = pCreate->daysPerFile, @@ -321,7 +361,13 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat mndSetDefaultDbCfg(&dbObj.cfg); - if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { + if (mndCheckDbName(dbObj.name, pUser) != 0) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + char errMsg[TSDB_ERROR_MSG_LEN] = {0}; + if (mndCheckDbCfg(pMnode, &dbObj.cfg, errMsg, TSDB_ERROR_MSG_LEN) != 0) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } @@ -403,11 +449,10 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser->acct); + int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser); mndReleaseUser(pMnode, pOperUser); if (code != 0) { - terrno = code; mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } @@ -415,7 +460,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mnodeSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { +static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { bool changed = false; if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { @@ -521,13 +566,14 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - int32_t code = mnodeSetDbCfgFromAlterDbMsg(&dbObj, pAlter); + int32_t code = mndSetDbCfgFromAlterDbMsg(&dbObj, pAlter); if (code != 0) { mndReleaseDb(pMnode, pDb); mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); return code; } + dbObj.version++; code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj); mndReleaseDb(pMnode, pDb); @@ -603,7 +649,6 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); if (code != 0) { - terrno = code; mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); return code; } @@ -772,10 +817,14 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe return 0; } -char *mnodeGetDbStr(char *src) { +char *mnGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); if (pos != NULL) ++pos; + if (pos == NULL) { + return src; + } + return pos; } @@ -794,7 +843,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char *name = mnodeGetDbStr(pDb->name); + char *name = mnGetDbStr(pDb->name); if (name != NULL) { STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); } else { @@ -887,7 +936,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 sdbRelease(pSdb, pDb); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4ce557fc02c6d0ebcc3ed76e9c25af37fab230b7..ec5f68a71372c0fd22c082d33d1ef5dac3b59c6c 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -19,7 +19,7 @@ #include "mndShow.h" #include "mndTrans.h" #include "ttime.h" -#include "tutil.h" +#include "tep.h" #define TSDB_DNODE_VER 1 #define TSDB_DNODE_RESERVE_SIZE 64 @@ -27,8 +27,6 @@ #define TSDB_CONIIG_VALUE_LEN 48 #define TSDB_CONFIG_NUMBER 8 -static int32_t id = 2; - static const char *offlineReason[] = { "", "status msg timeout", @@ -110,7 +108,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { } static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -177,6 +175,7 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) { mTrace("dnode:%d, perform update action", pOldDnode->id); + pOldDnode->updateTime = pNewDnode->updateTime; return 0; } @@ -389,7 +388,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { SDnodeObj dnodeObj = {0}; - dnodeObj.id = id++; + dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE); dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.updateTime = dnodeObj.createdTime; taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port); @@ -613,7 +612,7 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, cols++; } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } @@ -735,7 +734,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i sdbRelease(pSdb, pDnode); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 3fd7dcfba160d10d34e761faa61a032eaf8af6ba..b7158bb0947cc85e2700327421f32b1f2b5d7d2d 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -500,7 +500,7 @@ static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in sdbRelease(pSdb, pFunc); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index e91c51d301a3a9ecc81586d39761648ac975d858..7b6804d43e3c178aa6b3b46ccd681fb9dc127ec8 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -457,7 +457,7 @@ static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i sdbRelease(pSdb, pMnodeObj); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 59d84e5760206d8a331c23ab3a99dd3490d301e5..45a63f2dc57f26ebb0e338988b2a4971b81fc26c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -252,7 +252,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { +static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { pConn->numOfQueries = 0; pConn->numOfStreams = 0; int32_t numOfQueries = htonl(pMsg->numOfQueries); @@ -338,7 +338,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { return -1; } - mnodeSaveQueryStreamList(pConn, pReq); + mndSaveQueryStreamList(pConn, pReq); if (pConn->killed != 0) { pRsp->killConnection = 1; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index c2b2acfa0cd8dc86221e61b93225a465b8a09ca5..91a229956f290c520b41dba18fe2c9fea38caaca 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -296,6 +296,8 @@ char *mndShowStr(int32_t showType) { return "show streamtables"; case TSDB_MGMT_TABLE_TP: return "show topics"; + case TSDB_MGMT_TABLE_FUNCTION: + return "show functions"; default: return "undefined"; } @@ -308,7 +310,7 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow) { return false; } -void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { +void mndVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { if (rows < capacity) { for (int32_t i = 0; i < numOfCols; ++i) { memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 63bf186be5095f80f71a8071c3df7d3be2ed1a5e..791b6f5d12e682061979c1cca2d51a2ff55da9fd 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -69,7 +69,7 @@ int32_t mndInitStb(SMnode *pMnode) { void mndCleanupStb(SMnode *pMnode) {} static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { - int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size); if (pRaw == NULL) return NULL; @@ -605,7 +605,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM return 0; } -static void mnodeExtractTableName(char *tableId, char *name) { +static void mndExtractTableName(char *tableId, char *name) { int pos = -1; int num = 0; for (pos = 0; tableId[pos] != 0; ++pos) { @@ -665,7 +665,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 } pShow->numOfReads += numOfRows; - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9ab84d6557333d10c0e9e78d0b5af402b34929fd..32ac795301a31f8d793784467a0becbf0a7e3bb8 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -262,11 +262,6 @@ void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { sdbRelease(pSdb, pTrans); } -static int32_t mndGenerateTransId() { - static int32_t tmp = 0; - return ++tmp; -} - char *mndTransStageStr(ETrnStage stage) { switch (stage) { case TRN_STAGE_PREPARE: @@ -303,7 +298,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return NULL; } - pTrans->id = mndGenerateTransId(); + pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->rpcHandle = rpcHandle; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 452c14886d51b80b4f00893b167ff7ac1764dee0..7219342d599b2e9fe80bf95ba3f9ea716fcb8fc4 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -20,7 +20,8 @@ #include "mndTrans.h" #include "tkey.h" -#define SDB_USER_VER 1 +#define TSDB_USER_VER 1 +#define TSDB_USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); static SSdbRaw *mndUserActionEncode(SUserObj *pUser); @@ -93,7 +94,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { } static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, SDB_USER_VER, sizeof(SUserObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -103,6 +104,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) SDB_SET_INT8(pRaw, dataPos, pUser->superUser) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -112,7 +114,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_USER_VER) { + if (sver != TSDB_USER_VER) { mError("failed to decode user since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -129,6 +131,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superUser) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_USER_RESERVE_SIZE) return pRow; } @@ -166,12 +169,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) { mTrace("user:%s, perform update action", pOldUser->user); - memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN); memcpy(pOldUser->pass, pNewUser->pass, TSDB_PASSWORD_LEN); - memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN); - pOldUser->createdTime = pNewUser->createdTime; pOldUser->updateTime = pNewUser->updateTime; - pOldUser->superUser = pNewUser->superUser; return 0; } @@ -219,58 +218,6 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, return 0; } -static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); - if (pTrans == NULL) { - mError("user:%s, failed to update since %s", pOldUser->user, terrstr()); - return -1; - } - mDebug("trans:%d, used to update user:%s", pTrans->id, pOldUser->user); - - SSdbRaw *pRedoRaw = mndUserActionEncode(pNewUser); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - mndTransDrop(pTrans); - return 0; -} - -static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); - if (pTrans == NULL) { - mError("user:%s, failed to drop since %s", pUser->user, terrstr()); - return -1; - } - mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user); - - SSdbRaw *pRedoRaw = mndUserActionEncode(pUser); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - mndTransDrop(pTrans); - return 0; -} - static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; @@ -315,6 +262,32 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("user:%s, failed to update since %s", pOldUser->user, terrstr()); + return -1; + } + mDebug("trans:%d, used to update user:%s", pTrans->id, pOldUser->user); + + SSdbRaw *pRedoRaw = mndUserActionEncode(pNewUser); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; @@ -351,6 +324,7 @@ static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { memcpy(&newUser, pUser, sizeof(SUserObj)); memset(pUser->pass, 0, sizeof(pUser->pass)); taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); + newUser.updateTime = taosGetTimestampMs(); int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pMsg); sdbRelease(pMnode->pSdb, pOperUser); @@ -363,6 +337,32 @@ static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("user:%s, failed to drop since %s", pUser->user, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user); + + SSdbRaw *pRedoRaw = mndUserActionEncode(pUser); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SDropUserMsg *pDrop = pMsg->rpcMsg.pCont; @@ -486,7 +486,7 @@ static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in sdbRelease(pSdb, pUser); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 44d13d1fb498cb41a87fb69ab072d2e26be9895e..9a462c024b9344b0d73ecb21f86141f381c4372d 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -71,7 +71,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SDbObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -141,11 +141,8 @@ static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { mTrace("vgId:%d, perform update action", pOldVgroup->vgId); - pOldVgroup->vgId = pNewVgroup->vgId; - pOldVgroup->createdTime = pNewVgroup->createdTime; pOldVgroup->updateTime = pNewVgroup->updateTime; pOldVgroup->version = pNewVgroup->version; - memcpy(pOldVgroup->dbName, pNewVgroup->dbName, TSDB_FULL_DB_NAME_LEN); pOldVgroup->replica = pNewVgroup->replica; memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); return 0; @@ -284,7 +281,7 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, numOfRows++; } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } @@ -374,7 +371,7 @@ static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i sdbRelease(pSdb, pVgroup); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index e5434786083bf74a1546a1f1585b3fa2a17c8d79..24155a4e5329f73fc950e25ef23b942b55a11bf0 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -69,7 +69,7 @@ static void mndCleanupTimer(SMnode *pMnode) { } } -static int32_t mnodeCreateDir(SMnode *pMnode, const char *path) { +static int32_t mndCreateDir(SMnode *pMnode, const char *path) { pMnode->path = strdup(path); if (pMnode->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -245,7 +245,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } - int32_t code = mnodeCreateDir(pMnode, path); + int32_t code = mndCreateDir(pMnode, path); if (code != 0) { code = terrno; mError("failed to open mnode since %s", terrstr()); diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index a160533bf23b9141d9ab73d640f723f4369515a2..7e2e2602a88a944a7b2be8a2995d13f671cb4914 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -59,6 +59,7 @@ typedef struct SSdb { char *tmpDir; int64_t lastCommitVer; int64_t curVer; + int32_t maxId[SDB_MAX]; EKeyType keyTypes[SDB_MAX]; SHashObj *hashObjs[SDB_MAX]; SRWLatch locks[SDB_MAX]; diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index ccff5b6c82c773df7277e9c13c12f22f69ca2171..77614e399e1c8940b7b3eada86ce28b798f16732 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -127,6 +127,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { return -1; } + pSdb->maxId[sdbType] = 0; pSdb->hashObjs[sdbType] = hash; taosInitRWLatch(&pSdb->locks[sdbType]); mDebug("sdb table:%d is initialized", sdbType); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index c3874caffc7459376bdb0dbb33a4560deb81363f..f63f7236fbb56b0edfc49b61b6a2b00736454a65 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -72,6 +72,10 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosWUnLockLatch(pLock); + if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { + pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); + } + SdbInsertFp insertFp = pSdb->insertFps[pRow->type]; if (insertFp != NULL) { code = (*insertFp)(pSdb, pRow->pObj); @@ -290,3 +294,28 @@ int32_t sdbGetSize(SSdb *pSdb, ESdbType type) { return size; } + +int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { + SHashObj *hash = sdbGetHash(pSdb, type); + if (hash == NULL) return -1; + + if (pSdb->keyTypes[type] != SDB_KEY_INT32) return -1; + + int32_t maxId = 0; + + SRWLatch *pLock = &pSdb->locks[type]; + taosRLockLatch(pLock); + + SSdbRow **ppRow = taosHashIterate(hash, NULL); + while (ppRow != NULL) { + SSdbRow *pRow = *ppRow; + int32_t id = *(int32_t *)pRow->pObj; + maxId = MAX(id, maxId); + ppRow = taosHashIterate(hash, ppRow); + } + + taosRUnLockLatch(pLock); + + maxId = MAX(maxId, pSdb->maxId[type]); + return maxId + 1; +} diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 6972605afd9c1dd93b7fd0b3198c3837b5351f88..3623516624e8ea31c3eedd0bcdfa52cf20d16ef4 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - add_subdirectory(test) + #add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index cdeb89493210ccbb25af5dcefbcbffdab8ce7050..85460e8d912e0e896bdad6dc17a95199d13855d8 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -72,7 +72,9 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { } break; case TSDB_MSG_TYPE_SUBMIT: - /* code */ + if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) { + // TODO: handle error + } break; default: break; diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index a25b04e16108c71a5d017f3b25fac8730cbd88fe..f0bca3aa2ef95c67fd3aaf92045db10059cd4a0b 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -92,6 +92,8 @@ TEST(vnodeApiTest, test_create_table_encode_and_decode_function) { #endif TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { + vnodeDestroy("vnode1"); + GTEST_ASSERT_GE(vnodeInit(2), 0); // Create and open a vnode diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/tq/inc/tqMetaStore.h index b9e702a89a57cfdc4300e74be11e7235df6f850e..4190125e44d1e7ee4c014b6e531b027afd40a5cc 100644 --- a/source/dnode/vnode/tq/inc/tqMetaStore.h +++ b/source/dnode/vnode/tq/inc/tqMetaStore.h @@ -24,29 +24,29 @@ extern "C" { #endif -TqMetaStore* tqStoreOpen(const char* path, +STqMetaStore* tqStoreOpen(const char* path, TqSerializeFun pSerializer, TqDeserializeFun pDeserializer, TqDeleteFun pDeleter, int32_t tqConfigFlag ); -int32_t tqStoreClose(TqMetaStore*); +int32_t tqStoreClose(STqMetaStore*); //int32_t tqStoreDelete(TqMetaStore*); //int32_t tqStoreCommitAll(TqMetaStore*); -int32_t tqStorePersist(TqMetaStore*); +int32_t tqStorePersist(STqMetaStore*); //clean deleted idx and data from persistent file -int32_t tqStoreCompact(TqMetaStore*); +int32_t tqStoreCompact(STqMetaStore*); -void* tqHandleGet(TqMetaStore*, int64_t key); +void* tqHandleGet(STqMetaStore*, int64_t key); //make it unpersist -void* tqHandleTouchGet(TqMetaStore*, int64_t key); -int32_t tqHandleMovePut(TqMetaStore*, int64_t key, void* value); -int32_t tqHandleCopyPut(TqMetaStore*, int64_t key, void* value, size_t vsize); +void* tqHandleTouchGet(STqMetaStore*, int64_t key); +int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value); +int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); //delete committed kv pair //notice that a delete action still needs to be committed -int32_t tqHandleDel(TqMetaStore*, int64_t key); -int32_t tqHandleCommit(TqMetaStore*, int64_t key); -int32_t tqHandleAbort(TqMetaStore*, int64_t key); +int32_t tqHandleDel(STqMetaStore*, int64_t key); +int32_t tqHandleCommit(STqMetaStore*, int64_t key); +int32_t tqHandleAbort(STqMetaStore*, int64_t key); #ifdef __cplusplus } diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index c1a46e567b9ce446abde15bde9e12f1f52843ba9..5888141c58a1c84c6d98ddcfd04a49a14b0aea8a 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -49,8 +49,8 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAl pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->tqLogReader = tqLogReader; - // pTq->tqMemRef.pAlloctorFactory = allocFac; - // pTq->tqMemRef.pAllocator = allocFac->create(allocFac); + pTq->tqMemRef.pAlloctorFactory = allocFac; + pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if(pTq->tqMemRef.pAllocator == NULL) { //TODO } @@ -202,7 +202,6 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) { return totSize; } - TqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; } diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/tq/src/tqMetaStore.c index 81b48191bcdbe3b03d2267f0728427cd90299b58..3f40c94f244518a76ad42725c8067bb98c54c34d 100644 --- a/source/dnode/vnode/tq/src/tqMetaStore.c +++ b/source/dnode/vnode/tq/src/tqMetaStore.c @@ -22,11 +22,10 @@ #define TQ_META_NAME "tq.meta" #define TQ_IDX_NAME "tq.idx" +static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value); +static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key); -static int32_t tqHandlePutCommitted(TqMetaStore*, int64_t key, void* value); -static void* tqHandleGetUncommitted(TqMetaStore*, int64_t key); - -static inline void tqLinkUnpersist(TqMetaStore *pMeta, TqMetaList* pNode) { +static inline void tqLinkUnpersist(STqMetaStore *pMeta, STqMetaList* pNode) { if(pNode->unpersistNext == NULL) { pNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pNode->unpersistPrev = pMeta->unpersistHead; @@ -68,18 +67,18 @@ static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) { return lseek(fd, offset, SEEK_SET); } -TqMetaStore* tqStoreOpen(const char* path, +STqMetaStore* tqStoreOpen(const char* path, TqSerializeFun serializer, TqDeserializeFun deserializer, TqDeleteFun deleter, int32_t tqConfigFlag ) { - TqMetaStore* pMeta = malloc(sizeof(TqMetaStore)); + STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); if(pMeta == NULL) { //close return NULL; } - memset(pMeta, 0, sizeof(TqMetaStore)); + memset(pMeta, 0, sizeof(STqMetaStore)); //concat data file name and index file name size_t pathLen = strlen(path); @@ -105,14 +104,14 @@ TqMetaStore* tqStoreOpen(const char* path, } pMeta->idxFd = idxFd; - pMeta->unpersistHead = malloc(sizeof(TqMetaList)); + pMeta->unpersistHead = malloc(sizeof(STqMetaList)); if(pMeta->unpersistHead == NULL) { ASSERT(false); //close file //free memory return NULL; } - memset(pMeta->unpersistHead, 0, sizeof(TqMetaList)); + memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; @@ -149,11 +148,11 @@ TqMetaStore* tqStoreOpen(const char* path, ASSERT(idxBuf.head.writeOffset == idxRead); //loop read every entry for(int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - TqMetaList *pNode = malloc(sizeof(TqMetaList)); + STqMetaList *pNode = malloc(sizeof(STqMetaList)); if(pNode == NULL) { //TODO: free memory and return error } - memset(pNode, 0, sizeof(TqMetaList)); + memset(pNode, 0, sizeof(STqMetaList)); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); lseek(fileFd, pNode->handle.offset, SEEK_SET); @@ -199,7 +198,7 @@ TqMetaStore* tqStoreOpen(const char* path, //put into list int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; - TqMetaList* pBucketNode = pMeta->bucket[bucketKey]; + STqMetaList* pBucketNode = pMeta->bucket[bucketKey]; if(pBucketNode == NULL) { pMeta->bucket[bucketKey] = pNode; } else if(pBucketNode->handle.key == pNode->handle.key) { @@ -212,7 +211,7 @@ TqMetaStore* tqStoreOpen(const char* path, } if(pBucketNode->next) { ASSERT(pBucketNode->next->handle.key == pNode->handle.key); - TqMetaList *pNodeFound = pBucketNode->next; + STqMetaList *pNodeFound = pBucketNode->next; pNode->next = pNodeFound->next; pBucketNode->next = pNode; pBucketNode = pNodeFound; @@ -239,7 +238,7 @@ TqMetaStore* tqStoreOpen(const char* path, return pMeta; } -int32_t tqStoreClose(TqMetaStore* pMeta) { +int32_t tqStoreClose(STqMetaStore* pMeta) { //commit data and idx tqStorePersist(pMeta); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next==NULL); @@ -247,7 +246,7 @@ int32_t tqStoreClose(TqMetaStore* pMeta) { close(pMeta->idxFd); //free memory for(int i = 0; i < TQ_BUCKET_SIZE; i++) { - TqMetaList* pNode = pMeta->bucket[i]; + STqMetaList* pNode = pMeta->bucket[i]; while(pNode) { ASSERT(pNode->unpersistNext == NULL); ASSERT(pNode->unpersistPrev == NULL); @@ -259,7 +258,7 @@ int32_t tqStoreClose(TqMetaStore* pMeta) { && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } - TqMetaList* next = pNode->next; + STqMetaList* next = pNode->next; free(pNode); pNode = next; } @@ -270,12 +269,12 @@ int32_t tqStoreClose(TqMetaStore* pMeta) { return 0; } -int32_t tqStoreDelete(TqMetaStore* pMeta) { +int32_t tqStoreDelete(STqMetaStore* pMeta) { close(pMeta->fileFd); close(pMeta->idxFd); //free memory for(int i = 0; i < TQ_BUCKET_SIZE; i++) { - TqMetaList* pNode = pMeta->bucket[i]; + STqMetaList* pNode = pMeta->bucket[i]; pMeta->bucket[i] = NULL; while(pNode) { if(pNode->handle.valueInTxn @@ -286,7 +285,7 @@ int32_t tqStoreDelete(TqMetaStore* pMeta) { && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } - TqMetaList* next = pNode->next; + STqMetaList* next = pNode->next; free(pNode); pNode = next; } @@ -299,11 +298,11 @@ int32_t tqStoreDelete(TqMetaStore* pMeta) { } //TODO: wrap in tfile -int32_t tqStorePersist(TqMetaStore* pMeta) { +int32_t tqStorePersist(STqMetaStore* pMeta) { TqIdxPageBuf idxBuf; int64_t* bufPtr = (int64_t*)idxBuf.buffer; - TqMetaList *pHead = pMeta->unpersistHead; - TqMetaList *pNode = pHead->unpersistNext; + STqMetaList *pHead = pMeta->unpersistHead; + STqMetaList *pNode = pHead->unpersistNext; TqSerializedHead *pSHead = malloc(sizeof(TqSerializedHead)); if(pSHead == NULL) { //TODO: memory error @@ -384,11 +383,11 @@ int32_t tqStorePersist(TqMetaStore* pMeta) { pNode->handle.valueInTxn == NULL ) { int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; - TqMetaList* pBucketHead = pMeta->bucket[bucketKey]; + STqMetaList* pBucketHead = pMeta->bucket[bucketKey]; if(pBucketHead == pNode) { pMeta->bucket[bucketKey] = pNode->next; } else { - TqMetaList* pBucketNode = pBucketHead; + STqMetaList* pBucketNode = pBucketHead; while(pBucketNode->next != NULL && pBucketNode->next != pNode) { pBucketNode = pBucketNode->next; @@ -415,9 +414,9 @@ int32_t tqStorePersist(TqMetaStore* pMeta) { return 0; } -static int32_t tqHandlePutCommitted(TqMetaStore* pMeta, int64_t key, void* value) { +static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) { int64_t bucketKey = key & TQ_BUCKET_MASK; - TqMetaList* pNode = pMeta->bucket[bucketKey]; + STqMetaList* pNode = pMeta->bucket[bucketKey]; while(pNode) { if(pNode->handle.key == key) { //TODO: think about thread safety @@ -432,12 +431,12 @@ static int32_t tqHandlePutCommitted(TqMetaStore* pMeta, int64_t key, void* value pNode = pNode->next; } } - TqMetaList *pNewNode = malloc(sizeof(TqMetaList)); + STqMetaList *pNewNode = malloc(sizeof(STqMetaList)); if(pNewNode == NULL) { //TODO: memory error return -1; } - memset(pNewNode, 0, sizeof(TqMetaList)); + memset(pNewNode, 0, sizeof(STqMetaList)); pNewNode->handle.key = key; pNewNode->handle.valueInUse = value; //put into unpersist list @@ -448,9 +447,9 @@ static int32_t tqHandlePutCommitted(TqMetaStore* pMeta, int64_t key, void* value return 0; } -void* tqHandleGet(TqMetaStore* pMeta, int64_t key) { +void* tqHandleGet(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; - TqMetaList* pNode = pMeta->bucket[bucketKey]; + STqMetaList* pNode = pMeta->bucket[bucketKey]; while(pNode) { if(pNode->handle.key == key) { if(pNode->handle.valueInUse != NULL @@ -466,9 +465,9 @@ void* tqHandleGet(TqMetaStore* pMeta, int64_t key) { return NULL; } -void* tqHandleTouchGet(TqMetaStore* pMeta, int64_t key) { +void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; - TqMetaList* pNode = pMeta->bucket[bucketKey]; + STqMetaList* pNode = pMeta->bucket[bucketKey]; while(pNode) { if(pNode->handle.key == key) { if(pNode->handle.valueInUse != NULL @@ -485,9 +484,9 @@ void* tqHandleTouchGet(TqMetaStore* pMeta, int64_t key) { return NULL; } -static inline int32_t tqHandlePutImpl(TqMetaStore* pMeta, int64_t key, void* value) { +static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) { int64_t bucketKey = key & TQ_BUCKET_MASK; - TqMetaList* pNode = pMeta->bucket[bucketKey]; + STqMetaList* pNode = pMeta->bucket[bucketKey]; while(pNode) { if(pNode->handle.key == key) { //TODO: think about thread safety @@ -506,12 +505,12 @@ static inline int32_t tqHandlePutImpl(TqMetaStore* pMeta, int64_t key, void* val pNode = pNode->next; } } - TqMetaList *pNewNode = malloc(sizeof(TqMetaList)); + STqMetaList *pNewNode = malloc(sizeof(STqMetaList)); if(pNewNode == NULL) { //TODO: memory error return -1; } - memset(pNewNode, 0, sizeof(TqMetaList)); + memset(pNewNode, 0, sizeof(STqMetaList)); pNewNode->handle.key = key; pNewNode->handle.valueInTxn = value; pNewNode->next = pMeta->bucket[bucketKey]; @@ -520,11 +519,11 @@ static inline int32_t tqHandlePutImpl(TqMetaStore* pMeta, int64_t key, void* val return 0; } -int32_t tqHandleMovePut(TqMetaStore* pMeta, int64_t key, void* value) { +int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); } -int32_t tqHandleCopyPut(TqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { +int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { void *vmem = malloc(vsize); if(vmem == NULL) { //TODO: memory error @@ -534,9 +533,9 @@ int32_t tqHandleCopyPut(TqMetaStore* pMeta, int64_t key, void* value, size_t vsi return tqHandlePutImpl(pMeta, key, vmem); } -static void* tqHandleGetUncommitted(TqMetaStore* pMeta, int64_t key) { +static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; - TqMetaList* pNode = pMeta->bucket[bucketKey]; + STqMetaList* pNode = pMeta->bucket[bucketKey]; while(pNode) { if(pNode->handle.key == key) { if(pNode->handle.valueInTxn != NULL @@ -552,9 +551,9 @@ static void* tqHandleGetUncommitted(TqMetaStore* pMeta, int64_t key) { return NULL; } -int32_t tqHandleCommit(TqMetaStore* pMeta, int64_t key) { +int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; - TqMetaList* pNode = pMeta->bucket[bucketKey]; + STqMetaList* pNode = pMeta->bucket[bucketKey]; while(pNode) { if(pNode->handle.key == key) { if(pNode->handle.valueInTxn == NULL) { @@ -575,9 +574,9 @@ int32_t tqHandleCommit(TqMetaStore* pMeta, int64_t key) { return -2; } -int32_t tqHandleAbort(TqMetaStore* pMeta, int64_t key) { +int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; - TqMetaList* pNode = pMeta->bucket[bucketKey]; + STqMetaList* pNode = pMeta->bucket[bucketKey]; while(pNode) { if(pNode->handle.key == key) { if(pNode->handle.valueInTxn) { @@ -596,9 +595,9 @@ int32_t tqHandleAbort(TqMetaStore* pMeta, int64_t key) { return -2; } -int32_t tqHandleDel(TqMetaStore* pMeta, int64_t key) { +int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; - TqMetaList* pNode = pMeta->bucket[bucketKey]; + STqMetaList* pNode = pMeta->bucket[bucketKey]; while(pNode) { if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { if(pNode->handle.valueInTxn) { @@ -616,6 +615,6 @@ int32_t tqHandleDel(TqMetaStore* pMeta, int64_t key) { } //TODO: clean deleted idx and data from persistent file -int32_t tqStoreCompact(TqMetaStore *pMeta) { +int32_t tqStoreCompact(STqMetaStore *pMeta) { return 0; } diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/tsdb/inc/tsdbDef.h index 0a57f5f670e6d0544a2ec0978a2d2380d8d95229..7c593cb4c740385f8c1040e7019e80ef45952c45 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbDef.h +++ b/source/dnode/vnode/tsdb/inc/tsdbDef.h @@ -17,6 +17,10 @@ #define _TD_TSDB_DEF_H_ #include "mallocator.h" +#include "taosmsg.h" +#include "tdlist.h" +#include "thash.h" +#include "tskiplist.h" #include "tsdb.h" #include "tsdbMemTable.h" @@ -29,6 +33,8 @@ extern "C" { struct STsdb { char * path; STsdbCfg options; + STsdbMemTable * mem; + STsdbMemTable * imem; SMemAllocatorFactory *pmaf; }; diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h index 6be90ef53c86d4a489408e5cfae6c2a0d9855858..e7787af7cff24820b23ada5a4b7f2ea294fd3763 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h +++ b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h @@ -22,10 +22,11 @@ extern "C" { #endif -typedef struct SMemTable { - /* TODO */ - SMemAllocator *pma; -} SMemTable; +typedef struct STsdbMemTable STsdbMemTable; + +STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF); +void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable); +int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg); #ifdef __cplusplus } diff --git a/src/inc/tp.h b/source/dnode/vnode/tsdb/src/tsdbCommit.c similarity index 74% rename from src/inc/tp.h rename to source/dnode/vnode/tsdb/src/tsdbCommit.c index 1d6570898ebfcf0137fd2d502dc39de9378714b9..b12419773661e2aa97b94f4b26d6ba690b372120 100644 --- a/src/inc/tp.h +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -13,19 +13,9 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TP -#define TDENGINE_TP +#include "tsdbDef.h" -#ifdef __cplusplus -extern "C" { -#endif - -int32_t tpInit(); -void tpCleanUp(); -void tpUpdateTs(int32_t vgId, int64_t *seq, void *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif +int tsdbCommit(STsdb *pTsdb) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/tsdb/src/tsdbMemTable.c index 6dea4a4e57392be988126c579648f39a8270b9bf..e3d1f8673ed878f30021502e3ea6736861743831 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMemTable.c +++ b/source/dnode/vnode/tsdb/src/tsdbMemTable.c @@ -11,4 +11,84 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tsdbDef.h" + +#if 1 +typedef struct STbData { + TD_SLIST_NODE(STbData); + SSubmitMsg *pMsg; +} STbData; +#else +typedef struct STbData { + TD_SLIST_NODE(STbData); + uint64_t uid; // TODO: change here as tb_uid_t + TSKEY keyMin; + TSKEY keyMax; + uint64_t nRows; + SSkipList *pData; // Here need a container, may not use the SL + T_REF_DECLARE() +} STbData; +#endif + +struct STsdbMemTable { + T_REF_DECLARE() + SRWLatch latch; + TSKEY keyMin; + TSKEY keyMax; + uint64_t nRow; + SMemAllocator *pMA; + // Container + TD_SLIST(STbData) list; +}; + +STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { + STsdbMemTable *pMemTable; + SMemAllocator *pMA; + + pMA = (*pMAF->create)(pMAF); + ASSERT(pMA != NULL); + + pMemTable = (STsdbMemTable *)((*pMA->malloc)(pMA, sizeof(*pMemTable))); + if (pMemTable == NULL) { + (*pMAF->destroy)(pMAF, pMA); + return NULL; + } + + T_REF_INIT_VAL(pMemTable, 1); + taosInitRWLatch(&(pMemTable->latch)); + pMemTable->keyMin = TSKEY_MAX; + pMemTable->keyMax = TSKEY_MIN; + pMemTable->nRow = 0; + pMemTable->pMA = pMA; + tSListInit(&(pMemTable->list)); + + // TODO + return pMemTable; +} + +void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) { + SMemAllocator *pMA = pMemTable->pMA; + + if (pMA->free) { + // TODO + ASSERT(0); + } + + (*pMAF->destroy)(pMAF, pMA); +} + +int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) { + SMemAllocator *pMA = pMemTable->pMA; + STbData * pTbData = (STbData *)((*pMA->malloc)(pMA, sizeof(*pTbData))); + if (pTbData == NULL) { + // TODO + } + + tSListPush(&(pMemTable->list), pTbData); + + return 0; +} + +/* ------------------------ STATIC METHODS ------------------------ */ \ No newline at end of file diff --git a/src/inc/tmqtt.h b/source/dnode/vnode/tsdb/src/tsdbWrite.c similarity index 70% rename from src/inc/tmqtt.h rename to source/dnode/vnode/tsdb/src/tsdbWrite.c index 256e61fbae897cd77c1e7cc780c4c4c587293e28..f9441cbe44408ee4086ec86ef60e7b249723340c 100644 --- a/src/inc/tmqtt.h +++ b/source/dnode/vnode/tsdb/src/tsdbWrite.c @@ -13,20 +13,13 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MQTT_H -#define TDENGINE_MQTT_H +#include "tsdbDef.h" -#ifdef __cplusplus -extern "C" { -#endif - -int32_t mqttInitSystem(); -int32_t mqttStartSystem(); -void mqttStopSystem(); -void mqttCleanUpSystem(); - -#ifdef __cplusplus -} -#endif - -#endif +int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg) { + // Check if mem is there. If not, create one. + pTsdb->mem = tsdbNewMemTable(pTsdb->pmaf); + if (pTsdb->mem == NULL) { + return -1; + } + return tsdbInsertDataToMemTable(pTsdb->mem, pMsg); +} \ No newline at end of file diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a08b64f9a91b8db5b16bfae330dab3dd9b9388c8..455c82b1bc0e480d89136a37346f43d6ede319aa 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -24,16 +24,16 @@ extern "C" { #include "common.h" #include "tlog.h" -#define CTG_DEFAULT_CLUSTER_NUMBER 6 -#define CTG_DEFAULT_VGROUP_NUMBER 100 -#define CTG_DEFAULT_DB_NUMBER 20 +#define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 +#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 +#define CTG_DEFAULT_CACHE_DB_NUMBER 20 +#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 100000 #define CTG_DEFAULT_INVALID_VERSION (-1) typedef struct SVgroupListCache { int32_t vgroupVersion; - SHashObj *cache; // key:vgId, value:SVgroupInfo* - SArray *arrayCache; // SVgroupInfo + SHashObj *cache; // key:vgId, value:SVgroupInfo } SVgroupListCache; typedef struct SDBVgroupCache { @@ -41,20 +41,23 @@ typedef struct SDBVgroupCache { } SDBVgroupCache; typedef struct STableMetaCache { - SHashObj *cache; //key:fulltablename, value:STableMeta + SHashObj *cache; //key:fulltablename, value:STableMeta + SHashObj *stableCache; //key:suid, value:STableMeta* } STableMetaCache; typedef struct SCatalog { SVgroupListCache vgroupCache; - SDBVgroupCache dbCache; - STableMetaCache tableCache; + SDBVgroupCache dbCache; + STableMetaCache tableCache; } SCatalog; typedef struct SCatalogMgmt { void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata SHashObj *pCluster; // items cached for each cluster, the hash key is the cluster-id got from mgmt node + SCatalogCfg cfg; } SCatalogMgmt; +typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); extern int32_t ctgDebugFlag; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 92b60945291184afc39c151b6d7067acef8c8192..a670d9d6398320fe9e6da5b532a32340a88e8b3d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -16,6 +16,7 @@ #include "catalogInt.h" #include "trpc.h" #include "query.h" +#include "tname.h" SCatalogMgmt ctgMgmt = {0}; @@ -47,14 +48,14 @@ int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSe return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* exist) { - if (NULL == pCatalog->vgroupCache.arrayCache || pCatalog->vgroupCache.vgroupVersion < 0) { +int32_t ctgGetVgroupFromCache(struct SCatalog* pCatalog, SHashObj** pVgroupList, int32_t* exist) { + if (NULL == pCatalog->vgroupCache.cache || pCatalog->vgroupCache.vgroupVersion < 0) { *exist = 0; return TSDB_CODE_SUCCESS; } if (pVgroupList) { - *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache); + *pVgroupList = pCatalog->vgroupCache.cache; } *exist = 1; @@ -62,8 +63,7 @@ int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* return TSDB_CODE_SUCCESS; } - -int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { +int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { if (NULL == pCatalog->dbCache.cache) { *exist = 0; return TSDB_CODE_SUCCESS; @@ -92,6 +92,7 @@ int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, const char *dbName, SDBVgrou (*dbInfo)->vgroupVersion = info->vgroupVersion; (*dbInfo)->hashRange = info->hashRange; + (*dbInfo)->hashType = info->hashType; } *exist = 1; @@ -130,16 +131,362 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp } +int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, const char* pTableName, STableMeta** pTableMeta, int32_t *exist) { + if (NULL == pCatalog->tableCache.cache) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + + snprintf(tbFullName, sizeof(tbFullName), "%s.%s", dbName, pTableName); + + STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName)); + + if (NULL == tbMeta) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + if (tbMeta->tableType == TSDB_CHILD_TABLE) { + STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + if ((*stbMeta)->suid != tbMeta->suid) { + ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); + *pTableMeta = calloc(1, metaSize); + if (NULL == *pTableMeta) { + ctgError("calloc size[%d] failed", metaSize); + return TSDB_CODE_CTG_MEM_ERROR; + } + + memcpy(*pTableMeta, tbMeta, sizeof(SCTableMeta)); + memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + } else { + int32_t metaSize = sizeof(STableMeta) + (tbMeta->tableInfo.numOfTags + tbMeta->tableInfo.numOfColumns) * sizeof(SSchema); + *pTableMeta = calloc(1, metaSize); + if (NULL == *pTableMeta) { + ctgError("calloc size[%d] failed", metaSize); + return TSDB_CODE_CTG_MEM_ERROR; + } + + memcpy(*pTableMeta, tbMeta, metaSize); + } + + *exist = 1; + + return TSDB_CODE_SUCCESS; +} + +void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { + epSet->inUse = 0; + epSet->numOfEps = vgroupInfo->numOfEps; + + for (int32_t i = 0; i < vgroupInfo->numOfEps; ++i) { + memcpy(&epSet->port[i], &vgroupInfo->epAddr[i].port, sizeof(epSet->port[i])); + memcpy(&epSet->fqdn[i], &vgroupInfo->epAddr[i].fqdn, sizeof(epSet->fqdn[i])); + } +} + + +int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + + snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); + + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .tableFullName = tbFullName}; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_TABLE_META, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + SEpSet epSet; + + ctgGenEpSet(&epSet, vgroupInfo); + + rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp); + + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("get table meta from mnode failed, error code:%d", rpcRsp.code); + return rpcRsp.code; + } + + code = queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + return code; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) { + switch (hashType) { + default: + *fp = MurmurHash3_32; + break; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetVgroupFromVgId(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, int32_t vgId, SVgroupInfo *pVgroup) { + SHashObj *vgroupHash = NULL; + + CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); + if (NULL == vgroupHash) { + ctgError("get empty vgroup cache"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (NULL == taosHashGetClone(vgroupHash, &vgId, sizeof(vgId), pVgroup)) { + ctgError("vgId[%d] not found in vgroup list", vgId); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetVgroupFromVgIdBatch(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SArray* vgIds, SArray* vgroupList) { + SHashObj *vgroupHash = NULL; + SVgroupInfo pVgroup = {0}; + int32_t vgIdNum = taosArrayGetSize(vgIds); + + CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); + if (NULL == vgroupHash) { + ctgError("get empty vgroup cache"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + for (int32_t i = 0; i < vgIdNum; ++i) { + int32_t *vgId = taosArrayGet(vgIds, i); + + if (NULL == taosHashGetClone(vgroupHash, vgId, sizeof(*vgId), &pVgroup)) { + ctgError("vgId[%d] not found in vgroup list", vgId); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (NULL == taosArrayPush(vgroupList, &pVgroup)) { + ctgError("push vgroup to array failed, idx:%d", i); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { + SDBVgroupInfo *dbInfo = NULL; + int32_t code = 0; + + CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo)); + + if (NULL == dbInfo) { + ctgWarn("db[%s] vgroup info not found", pDBName); + return TSDB_CODE_TSC_DB_NOT_SELECTED; + } + + if (dbInfo->vgroupVersion < 0 || NULL == dbInfo->vgId) { + ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgId:%p", pDBName, dbInfo->vgroupVersion, dbInfo->vgId); + CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + int32_t vgNum = taosArrayGetSize(dbInfo->vgId); + if (vgNum <= 0) { + ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum); + CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + tableNameHashFp fp = NULL; + + CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashType, &fp)); + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + + snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); + + uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); + uint32_t hashUnit = dbInfo->hashRange / vgNum; + uint32_t vgId = hashValue / hashUnit; + + CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, vgId, pVgroup)); + +_return: + if (dbInfo && dbInfo->vgId) { + taosArrayDestroy(dbInfo->vgId); + dbInfo->vgId = NULL; + } + + tfree(dbInfo); + + return code; +} + + + +STableMeta* ctgCreateSTableMeta(STableMetaMsg* pChild) { + assert(pChild != NULL); + int32_t total = pChild->numOfColumns + pChild->numOfTags; + + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); + pTableMeta->tableType = TSDB_SUPER_TABLE; + pTableMeta->tableInfo.numOfTags = pChild->numOfTags; + pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; + pTableMeta->tableInfo.precision = pChild->precision; + + pTableMeta->uid = pChild->suid; + pTableMeta->tversion = pChild->tversion; + pTableMeta->sversion = pChild->sversion; + + memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total); + + int32_t num = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < num; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) { + if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + int32_t exist = 0; + + if (!forceUpdate) { + CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist)); + + if (exist) { + return TSDB_CODE_SUCCESS; + } + } + + CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName)); + + CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist)); + + if (0 == exist) { + ctgError("get table meta from cache failed, but fetch succeed"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { + if (output->metaNum != 1 && output->metaNum != 2) { + ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (NULL == output->tbMeta) { + ctgError("no valid table meta got from meta rsp"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (NULL == pCatalog->tableCache.cache) { + pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->tableCache.cache) { + ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } + + if (NULL == pCatalog->tableCache.cache) { + pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->tableCache.cache) { + ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + + pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->tableCache.stableCache) { + ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } + + if (output->metaNum == 2) { + if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { + ctgError("push ctable[%s] to table cache failed", output->ctbFname); + goto error_exit; + } + + if (TSDB_SUPER_TABLE != output->tbMeta->tableType) { + ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE); + goto error_exit; + } + } + + if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, sizeof(*output->tbMeta)) != 0) { + ctgError("push table[%s] to table cache failed", output->tbFname); + goto error_exit; + } + + if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { + if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &output->tbMeta, POINTER_BYTES) != 0) { + ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid); + goto error_exit; + } + } + + return TSDB_CODE_SUCCESS; + +error_exit: + if (pCatalog->vgroupCache.cache) { + taosHashCleanup(pCatalog->vgroupCache.cache); + pCatalog->vgroupCache.cache = NULL; + } + + pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + int32_t catalogInit(SCatalogCfg *cfg) { - ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == ctgMgmt.pCluster) { - CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER); + CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); + } + + if (cfg) { + memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg)); + } else { + ctgMgmt.cfg.enableVgroupCache = true; + ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; + ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; } return TSDB_CODE_SUCCESS; } - int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { if (NULL == clusterId || NULL == catalogHandle) { return TSDB_CODE_CTG_INVALID_INPUT; @@ -190,7 +537,7 @@ int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) { -int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { +int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { if (NULL == pVgroup) { ctgError("no valid vgroup list info to update"); return TSDB_CODE_CTG_INTERNAL_ERROR; @@ -200,22 +547,11 @@ int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion); return TSDB_CODE_CTG_INVALID_INPUT; } - - - if (NULL == pCatalog->vgroupCache.arrayCache) { - pCatalog->vgroupCache.arrayCache = taosArrayInit(pVgroup->vgroupNum, sizeof(pVgroup->vgroupInfo[0])); - if (NULL == pCatalog->vgroupCache.arrayCache) { - ctgError("init array[%d] for cluster cache failed", pVgroup->vgroupNum); - return TSDB_CODE_CTG_MEM_ERROR; - } - } else { - taosArrayClear(pCatalog->vgroupCache.arrayCache); - } if (NULL == pCatalog->vgroupCache.cache) { - pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_CACHE_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->vgroupCache.cache) { - ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_VGROUP_NUMBER); + ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_CACHE_VGROUP_NUMBER); return TSDB_CODE_CTG_MEM_ERROR; } } else { @@ -224,13 +560,7 @@ int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) SVgroupInfo *vInfo = NULL; for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) { - vInfo = taosArrayPush(pCatalog->vgroupCache.arrayCache, &pVgroup->vgroupInfo[i]); - if (NULL == vInfo) { - ctgError("push to vgroup array cache failed"); - goto error_exit; - } - - if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &vInfo, POINTER_BYTES) != 0) { + if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &pVgroup->vgroupInfo[i], sizeof(pVgroup->vgroupInfo[i])) != 0) { ctgError("push to vgroup hash cache failed"); goto error_exit; } @@ -241,11 +571,6 @@ int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) return TSDB_CODE_SUCCESS; error_exit: - if (pCatalog->vgroupCache.arrayCache) { - taosArrayDestroy(pCatalog->vgroupCache.arrayCache); - pCatalog->vgroupCache.arrayCache = NULL; - } - if (pCatalog->vgroupCache.cache) { taosHashCleanup(pCatalog->vgroupCache.cache); pCatalog->vgroupCache.cache = NULL; @@ -256,15 +581,14 @@ error_exit: return TSDB_CODE_CTG_INTERNAL_ERROR; } - -int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList) { +int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash) { if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) { return TSDB_CODE_CTG_INVALID_INPUT; } int32_t exist = 0; - CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist)); if (exist) { return TSDB_CODE_SUCCESS; @@ -274,10 +598,10 @@ int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup)); - CTG_ERR_RET(catalogUpdateVgroup(pCatalog, pVgroup)); + CTG_ERR_RET(catalogUpdateVgroupCache(pCatalog, pVgroup)); - if (pVgroupList) { - CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + if (pVgroupHash) { + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist)); } if (0 == exist) { @@ -288,6 +612,7 @@ int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, return TSDB_CODE_SUCCESS; } + int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { if (NULL == pCatalog || NULL == dbName || NULL == version) { return TSDB_CODE_CTG_INVALID_INPUT; @@ -309,7 +634,7 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, return TSDB_CODE_SUCCESS; } -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { +int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { return TSDB_CODE_CTG_INVALID_INPUT; } @@ -324,9 +649,9 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB } if (NULL == pCatalog->dbCache.cache) { - pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->dbCache.cache) { - ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_DB_NUMBER); + ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); return TSDB_CODE_CTG_MEM_ERROR; } } @@ -369,11 +694,11 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); if (DbOut.vgroupList) { - CTG_ERR_JRET(catalogUpdateVgroup(pCatalog, DbOut.vgroupList)); + CTG_ERR_JRET(catalogUpdateVgroupCache(pCatalog, DbOut.vgroupList)); } if (DbOut.dbVgroup) { - CTG_ERR_JRET(catalogUpdateDBVgroup(pCatalog, dbName, DbOut.dbVgroup)); + CTG_ERR_JRET(catalogUpdateDBVgroupCache(pCatalog, dbName, DbOut.dbVgroup)); } if (dbInfo) { @@ -388,47 +713,121 @@ _return: return code; } +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { + return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, false, pTableMeta); +} - -int32_t catalogGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta) { - if (NULL == pCatalog || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) { + if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) { return TSDB_CODE_CTG_INVALID_INPUT; } - SBuildTableMetaInput bInput = {0}; - char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; - int32_t msgLen = 0; - - int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); - if (code) { - return code; - } - - SRpcMsg rpcMsg = { - .msgType = TSDB_MSG_TYPE_TABLE_META, - .pCont = msg, - .contLen = msgLen, - }; + SVgroupInfo vgroupInfo = {0}; + + CTG_ERR_RET(ctgGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo)); - SRpcMsg rpcRsp = {0}; + STableMetaOutput output = {0}; + + CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output)); - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(ctgUpdateTableMetaCache(pCatalog, &output)); + tfree(output.tbMeta); + return TSDB_CODE_SUCCESS; } -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta) { +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { + return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta); +} +int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + STableMeta *tbMeta = NULL; + int32_t code = 0; + SVgroupInfo vgroupInfo = {0}; + SDBVgroupInfo *dbVgroup = NULL; + + CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta)); + + if (tbMeta->tableType == TSDB_SUPER_TABLE) { + CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup)); + + CTG_ERR_JRET(ctgGetVgroupFromVgIdBatch(pCatalog, pRpc, pMgmtEps, dbVgroup->vgId, pVgroupList)); + } else { + CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, tbMeta->vgId, &vgroupInfo)); + + if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) { + ctgError("push vgroupInfo to array failed"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + } + +_return: + tfree(tbMeta); + if (dbVgroup && dbVgroup->vgId) { + taosArrayDestroy(dbVgroup->vgId); + dbVgroup->vgId = NULL; + } + + tfree(dbVgroup); + + return code; } -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { - if (NULL == pCatalog || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { return TSDB_CODE_CTG_INVALID_INPUT; } + + int32_t code = 0; + + if (pReq->pTableName) { + char dbName[TSDB_FULL_DB_NAME_LEN]; + int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName); + if (tbNum > 0) { + pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES); + if (NULL == pRsp->pTableMeta) { + ctgError("taosArrayInit num[%d] failed", tbNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } + + for (int32_t i = 0; i < tbNum; ++i) { + SName *name = taosArrayGet(pReq->pTableName, i); + STableMeta *pTableMeta = NULL; + + snprintf(dbName, sizeof(dbName), "%s.%s", name->acctId, name->dbname); + + CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, dbName, name->tname, &pTableMeta)); + + if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { + ctgError("taosArrayPush failed, idx:%d", i); + tfree(pTableMeta); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + } + } + + return TSDB_CODE_SUCCESS; + +_return: + + if (pRsp->pTableMeta) { + int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); + for (int32_t i = 0; i < aSize; ++i) { + STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); + tfree(pMeta); + } + + taosArrayDestroy(pRsp->pTableMeta); + } - return 0; + return code; } void catalogDestroy(void) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0999c44fab3f85529c678bf1fc46f405de7a4e41..f37c7727f71af38a75ef59e86d0d2ffefd7566a9 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4831,8 +4831,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { STableBlockDist tableBlockDist = {0}; tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; - if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS; + if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { ++numRowSteps; } tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index a1d4962e8be815847ca1a25689588dcb157c318f..20037f829adae78ea866429650c06da6ec79202c 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -30,6 +30,7 @@ extern "C" { typedef struct Fst Fst; typedef struct FstNode FstNode; +typedef struct StreamWithState StreamWithState; typedef enum { Included, Excluded, Unbounded} FstBound; @@ -283,6 +284,9 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null); FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); +// into stream to expand later +StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb); + bool fstVerify(Fst *fst); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 37bdcb0ecfa15a0a2726de07d4a69470b44f4b0e..9cb4ac68363def9754380c548b6bf3323677f389 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1094,6 +1094,10 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } +StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb) { + if (sb == NULL) { return NULL; } + return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); +} FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } @@ -1119,7 +1123,7 @@ CompiledAddr fstGetRootAddr(Fst *fst) { Output fstEmptyFinalOutput(Fst *fst, bool *null) { Output res = 0; - FstNode *node = fst->root; + FstNode *node = fstGetRoot(fst); if (FST_NODE_IS_FINAL(node)) { *null = false; res = FST_NODE_FINAL_OUTPUT(node); @@ -1176,7 +1180,7 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { - return bound->type == Included ? true : false; + return bound->type == Excluded? false : true; } void fstBoundDestroy(FstBoundWithData *bound) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index d905147654593b74dc9898fb44c4838aa17927fb..f70b90041ba7a853f1cf936a594c496aec978a71 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -68,7 +68,7 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { // prefix query, impl later static void* prefixStart(AutomationCtx *ctx) { - StartWithStateValue *data = (StartWithStateValue *)(ctx->data); + StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata); return startWithStateValueDump(data); }; static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { @@ -145,7 +145,8 @@ AutomationCtx* automCtxCreate(void *data,AutomationType atype) { StartWithStateValue *sv = NULL; if (atype == AUTOMATION_PREFIX) { - sv = startWithStateValueCreate(Running, FST_INT, 0); + int val = 0; + sv = startWithStateValueCreate(Running, FST_INT, &val); ctx->stdata = (void *)sv; } else if (atype == AUTMMATION_MATCH) { diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 0cfb0fedc37706bfddebd7c025690baf380be014..f582536817ad8137ac32ad650bff5dbdb6ee76a1 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -59,9 +59,22 @@ class FstReadMemory { return ok; } // add later - bool Search(const std::string &key, std::vector &result) { + bool Search(AutomationCtx *ctx, std::vector &result) { + FstStreamBuilder *sb = fstSearch(_fst, ctx); + StreamWithState *st = streamBuilderIntoStream(sb); + StreamWithStateResult *rt = NULL; + + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + result.push_back((uint64_t)(rt->out.out)); + } return true; } + bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector &result) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Search(ctx, result); + int64_t e = taosGetTimestampUs(); + return ok; + } ~FstReadMemory() { fstCountingWriterDestroy(_w); @@ -186,11 +199,43 @@ void checkFstPerf() { printf("success to init fst read"); } Performance_fstReadRecords(m); - delete m; } +void checkFstPrefixSearch() { + FstWriter *fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int count = 2; + std::string key("ab"); + + for (int i = 0; i < count; i++) { + key[1] = key[1] + i; + fw->Put(key, i); + } + int64_t e = taosGetTimestampUs(); + + std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + delete fw; + FstReadMemory *m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + + // prefix search + std::vector result; + AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX); + m->Search(ctx, result); + assert(result.size() == count); + for (int i = 0; i < result.size(); i++) { + assert(result[i] == i); // check result + } + + free(ctx); + delete m; +} void validateFst() { int val = 100; int count = 100; @@ -209,6 +254,8 @@ void validateFst() { FstReadMemory *m = new FstReadMemory(1024 * 64); if (m->init() == false) { std::cout << "init readMemory failed" << std::endl; + delete m; + return; } { @@ -230,10 +277,12 @@ void validateFst() { } } delete m; - } + + int main(int argc, char** argv) { checkFstPerf(); + //checkFstPrefixSearch(); return 1; } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index d2813886b323c41658f07b50d46da7891d0f66d2..9f1363a9b401d8a50504c43ac6d8471ff2964be9 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4088,7 +4088,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } // load the meta data from catalog - code = catalogGetAllMeta(pCatalog, NULL, &req, &data); + code = catalogGetAllMeta(pCatalog, NULL, NULL, &req, &data); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 97102c5b00512f5c426121ee14720075458be7c1..f63f13d8ac180ddb93007d564cece710a7e4babc 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -71,7 +71,7 @@ typedef struct SInsertParseContext { const char* pSql; SMsgBuf msg; struct SCatalog* pCatalog; - STableMeta tableMeta; + STableMeta* pTableMeta; SHashObj* pTableBlockHashObj; // data block for each table. need release int32_t totalNum; SInsertStmtInfo* pOutput; @@ -186,7 +186,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0}; char tableName[TSDB_TABLE_NAME_LEN] = {0}; CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName)); - CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, &pCxt->tableMeta)); + CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); return TSDB_CODE_SUCCESS; } @@ -645,13 +645,13 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); CHECK_CODE(getTableMeta(pCxt, &sToken)); - if (TSDB_SUPER_TABLE != pCxt->tableMeta.tableType) { + if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } - SSchema* pTagsSchema = getTableTagSchema(&pCxt->tableMeta); + SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); SParsedDataColInfo spd = {0}; - setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(&pCxt->tableMeta)); + setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); @@ -668,7 +668,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(&pCxt->tableMeta).precision)); + CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); return TSDB_CODE_SUCCESS; } @@ -810,12 +810,12 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } STableDataBlocks *dataBuf = NULL; - CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->tableMeta.uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(&pCxt->tableMeta).rowSize, &pCxt->tableMeta, &dataBuf, NULL)); + CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); if (TK_LP == sToken.type) { // pSql -> field1_name, ...) - CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(&pCxt->tableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); + CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -861,7 +861,7 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { .pSql = pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, .pCatalog = NULL, - .tableMeta = {0}, + .pTableMeta = NULL, .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .totalNum = 0, .pOutput = *pInfo diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 28d01b9e66f35d38abca57a7fbe9504e61ee9647..7fcb7ea3047d1baaece2c5b7c9e89d83d66012e8 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1464,29 +1464,6 @@ int32_t copyTagData(STagData* dst, const STagData* src) { return 0; } -STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { - assert(pChild != NULL); - int32_t total = pChild->numOfColumns + pChild->numOfTags; - - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); - pTableMeta->tableType = TSDB_SUPER_TABLE; - pTableMeta->tableInfo.numOfTags = pChild->numOfTags; - pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; - pTableMeta->tableInfo.precision = pChild->precision; - - pTableMeta->uid = pChild->suid; - pTableMeta->tversion = pChild->tversion; - pTableMeta->sversion = pChild->sversion; - - memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total); - - int32_t num = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < num; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} uint32_t getTableMetaSize(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); diff --git a/source/libs/query/src/querymsg.c b/source/libs/query/src/querymsg.c index 8f35fd9c3ef43732dfae9c19423baa91306cdc13..c5864fd41b5ed9e2f632ab2d43d73cde660da25f 100644 --- a/source/libs/query/src/querymsg.c +++ b/source/libs/query/src/querymsg.c @@ -222,6 +222,13 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { pOut->dbVgroup->vgroupVersion = pRsp->dbVgroupVersion; pOut->dbVgroup->hashRange = htonl(pRsp->dbHashRange); + pOut->dbVgroup->hashType = htonl(pRsp->dbHashType); + + if (pOut->dbVgroup->hashRange < 0) { + qError("invalid hashRange[%d] for db[%s]", pOut->dbVgroup->hashRange, pRsp->db); + code = TSDB_CODE_TSC_INVALID_INPUT; + goto _exit; + } for (int32_t i = 0; i < pRsp->dbVgroupNum; ++i) { *(vgIdList + i) = htonl(*(vgIdList + i)); @@ -244,13 +251,134 @@ _exit: return code; } +static int32_t queryConvertTableMetaMsg(STableMetaMsg* pMetaMsg) { + pMetaMsg->numOfTags = htonl(pMetaMsg->numOfTags); + pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); + pMetaMsg->sversion = htonl(pMetaMsg->sversion); + pMetaMsg->tversion = htonl(pMetaMsg->tversion); + pMetaMsg->tuid = htobe64(pMetaMsg->tuid); + pMetaMsg->suid = htobe64(pMetaMsg->suid); + pMetaMsg->vgId = htonl(pMetaMsg->vgId); + + if (pMetaMsg->numOfTags < 0 || pMetaMsg->numOfTags > TSDB_MAX_TAGS) { + qError("invalid numOfTags[%d] in table meta rsp msg", pMetaMsg->numOfTags); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { + qError("invalid numOfColumns[%d] in table meta rsp msg", pMetaMsg->numOfColumns); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->tableType != TSDB_SUPER_TABLE && pMetaMsg->tableType != TSDB_CHILD_TABLE && pMetaMsg->tableType != TSDB_NORMAL_TABLE) { + qError("invalid tableType[%d] in table meta rsp msg", pMetaMsg->tableType); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->sversion < 0) { + qError("invalid sversion[%d] in table meta rsp msg", pMetaMsg->sversion); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->tversion < 0) { + qError("invalid tversion[%d] in table meta rsp msg", pMetaMsg->tversion); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + SSchema* pSchema = pMetaMsg->pSchema; + + int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; + for (int i = 0; i < numOfTotalCols; ++i) { + pSchema->bytes = htonl(pSchema->bytes); + pSchema->colId = htonl(pSchema->colId); + + pSchema++; + } + + if (pMetaMsg->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchema[0].colId); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t queryCreateTableMetaFromMsg(STableMetaMsg* msg, bool isSuperTable, STableMeta **pMeta) { + int32_t total = msg->numOfColumns + msg->numOfTags; + int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; + + STableMeta* pTableMeta = calloc(1, metaSize); + if (NULL == pTableMeta) { + qError("calloc size[%d] failed", metaSize); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pTableMeta->tableType = isSuperTable ? TSDB_SUPER_TABLE : msg->tableType; + pTableMeta->uid = msg->suid; + pTableMeta->suid = msg->suid; + pTableMeta->sversion = msg->sversion; + pTableMeta->tversion = msg->tversion; + + pTableMeta->tableInfo.numOfTags = msg->numOfTags; + pTableMeta->tableInfo.precision = msg->precision; + pTableMeta->tableInfo.numOfColumns = msg->numOfColumns; + + for(int32_t i = 0; i < msg->numOfColumns; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + memcpy(pTableMeta->schema, msg->pSchema, sizeof(SSchema) * total); + + *pMeta = pTableMeta; + + return TSDB_CODE_SUCCESS; +} + + +int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)msg; + int32_t code = queryConvertTableMetaMsg(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STableMetaOutput *pOut = (STableMetaOutput *)output; + + if (!tIsValidSchema(pMetaMsg->pSchema, pMetaMsg->numOfColumns, pMetaMsg->numOfTags)) { + qError("validate table meta schema in rsp msg failed"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { + pOut->metaNum = 2; + + memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname)); + memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname)); + + pOut->ctbMeta.vgId = pMetaMsg->vgId; + pOut->ctbMeta.tableType = pMetaMsg->tableType; + pOut->ctbMeta.uid = pMetaMsg->tuid; + pOut->ctbMeta.suid = pMetaMsg->suid; + + code = queryCreateTableMetaFromMsg(pMetaMsg, true, &pOut->tbMeta); + } else { + pOut->metaNum = 1; + + memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); + + code = queryCreateTableMetaFromMsg(pMetaMsg, false, &pOut->tbMeta); + } + + return code; +} + void msgInit() { queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg; queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg; queryBuildMsg[TSDB_MSG_TYPE_USE_DB] = queryBuildUseDbMsg; - //tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = tscProcessTableMetaRsp; + queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = queryProcessTableMetaRsp; queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp; queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB] = queryProcessUseDBRsp; diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 48142878c3bb7ba911582a15dda3986714e364aa..819afcc411b778ab57159a931116118d937208bb 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -105,6 +105,10 @@ static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) { return taosCalcChecksum(0, (uint8_t*)body, len); } +static inline int64_t walGetVerIdxOffset(SWal* pWal, int64_t ver) { + return (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry); +} + static inline void walResetVer(SWalVer* pVer) { pVer->firstVer = -1; pVer->verInSnapshotting = -1; @@ -117,6 +121,10 @@ int walLoadMeta(SWal* pWal); int walSaveMeta(SWal* pWal); int walRollFileInfo(SWal* pWal); +int walCheckAndRepairMeta(SWal* pWal); + +int walCheckAndRepairIdx(SWal* pWal); + char* walMetaSerialize(SWal* pWal); int walMetaDeserialize(SWal* pWal, const char* bytes); //meta section end diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index aa592b4fe8cded905de7bd56fd09f0c0e52254f1..e2715d51f1cab93601613bc7cefb7ce46ec29858 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -40,6 +40,47 @@ static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } +int walCheckAndRepairMeta(SWal* pWal) { + // load log files, get first/snapshot/last version info + const char* logPattern = "^[0-9]+.log$"; + const char* idxPattern = "^[0-9]+.idx$"; + regex_t logRegPattern; + regex_t idxRegPattern; + SArray* pLogArray = taosArrayInit(8, sizeof(int64_t)); + + regcomp(&logRegPattern, logPattern, REG_EXTENDED); + regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); + + DIR *dir = opendir(pWal->path); + if(dir == NULL) { + wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); + return -1; + } + + struct dirent* ent; + while((ent = readdir(dir)) != NULL) { + char *name = basename(ent->d_name); + int code = regexec(&logRegPattern, name, 0, NULL, 0); + if(code == 0) { + int64_t firstVer; + sscanf(name, "%" PRId64 ".log", &firstVer); + taosArrayPush(pLogArray, &firstVer); + } + } + + + // load meta + // if not match, or meta missing + // rebuild meta + return 0; +} + +int walCheckAndRepairIdx(SWal* pWal) { + // iterate all idx files + // check first and last entry of each idx file valid + return 0; +} + int walRollFileInfo(SWal* pWal) { int64_t ts = taosGetTimestampSec(); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 9efeb83cf04d539ab13e2bffc768e0e50432ea9a..06df3eb3be9360b407eb749a97c6242ca3074e96 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -90,6 +90,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } //open meta + walResetVer(&pWal->vers); pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; pWal->writeCur = -1; @@ -101,7 +102,6 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } //init status - walResetVer(&pWal->vers); pWal->totSize = 0; pWal->lastRollSeq = -1; @@ -123,7 +123,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - if(walLoadMeta(pWal) < 0) { + if(walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) { taosRemoveRef(tsWal.refSetId, pWal->refId); pthread_mutex_destroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); @@ -131,6 +131,10 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } + if(walCheckAndRepairIdx(pWal) < 0) { + + } + wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); return pWal; diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 7db5b90c1db869697b85c0550b88344c6e17b912..769b2fd2e4e48126c5bdd0426b93ab5404afacab 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -27,20 +27,20 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { int64_t logTfd = pWal->writeLogTfd; //seek position - int64_t offset = (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry); - code = tfLseek(idxTfd, offset, SEEK_SET); + int64_t idxOff = walGetVerIdxOffset(pWal, ver); + code = tfLseek(idxTfd, idxOff, SEEK_SET); if(code != 0) { return -1; } - int64_t readBuf[2]; - code = tfRead(idxTfd, readBuf, sizeof(readBuf)); + WalIdxEntry entry; + //TODO:deserialize + code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry)); if(code != 0) { return -1; } - //TODO:deserialize - ASSERT(readBuf[0] == ver); - code = tfLseek(logTfd, readBuf[1], SEEK_CUR); - if (code != 0) { + ASSERT(entry.ver == ver); + code = tfLseek(logTfd, entry.offset, SEEK_CUR); + if (code < 0) { return -1; } return code; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ffbb19c6b7a8b293dd3d0f7a29d06a6ec2fc30e8..67b70f249d0184b60acadc551569df9adceb426d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -68,13 +68,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) { walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); int64_t idxTfd = tfOpenReadWrite(fnameStr); - //change to deserialize function - + //TODO:change to deserialize function if(idxTfd < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; } - int idxOff = (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry); + int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = tfLseek(idxTfd, idxOff, SEEK_SET); if(code < 0) { pthread_mutex_unlock(&pWal->mutex); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 42fde042e7a56e94b9afb7c6e57015a698614c13..70a3dc622fdda1707fcefbbaa6d7cacb16890b39 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -129,7 +129,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data typ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") -// mnode +// mnode-common +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Message need to be reprocessed") @@ -140,7 +141,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION, "Incompatible protocol TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN, "Invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE, "Invalid message type") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS, "Too many connections") + +// mnode-show TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") + +// mnode-profile TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, "Invalid stream id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, "Invalid connection id") @@ -150,6 +155,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_START_SYNC, "failed to start sync" TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CREATE_DIR, "failed to create mnode dir") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_INIT_STEP, "failed to init components") +// mnode-sdb TAOS_DEFINE_ERROR(TSDB_CODE_SDB_APP_ERROR, "Unexpected generic error in sdb") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_ALREADY_THERE, "Object already there") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_NOT_THERE, "Object not there") @@ -164,36 +170,50 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_LEN, "Invalid raw data len" TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_CONTENT, "Invalid raw data content") // mnode-dnode -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "DNode already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "DNode does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of DNodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "Dnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "Dnode does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DNODES, "Too many dnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of dnodes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_CFG, "Cluster cfg inconsistent") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, "Cluster id not match") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG, "Invalid dnode cfg") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_EP, "Invalid dnode end point") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID, "Invalid dnode id") -// mnode-vgroup -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") +// mnode-mnode +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "Too many mnodes") // mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_NOT_EXIST, "Invalid account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_ACCTS, "Too many accounts") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, "Invalid account options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, "Account authorization has expired") +// mnode-user TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, "User already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_NOT_EXIST, "Invalid user") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT, "Invalid user format") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT, "Invalid password format") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, "Can not get user from conn") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") +// mnode-db +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_ACCT, "Invalid database account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") + +// mnode-vgroup +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") // mnode-stable TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") @@ -203,7 +223,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, "Too many time series") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") // operation only available for super table +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, "Tag name too long") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, "Tag already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") @@ -224,31 +244,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_SIZE, "Invalid database cache block size option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS, "Invalid database total blocks option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_DAYS, "Invalid database days option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP0, "Invalid database keep0 option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP1, "Invalid database keep1 option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP2, "Invalid database keep2 option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_MIN_ROWS, "Invalid database min rows option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_MAX_ROWS, "Invalid database max rows option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_COMMIT_TIME, "Invalid database commit time option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD, "Invalid database fsync periodoptions") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_WAL_LEVEL, "Invalid database wal level option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_PRECISION, "Invalid database precisin option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_COMP, "Invalid database compression option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_REPLICA, "Invalid database replication option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_QUORUM, "Invalid database quorum option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_UPDATE, "Invalid database update option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_LAST, "Invalid database cache last option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") - // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2d7179c0d319c8abad0a58a89e4456cb3fce33b1..23a923a315765778690c5ba72400148d99527792 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -81,15 +81,11 @@ int32_t writeInterval = DEFAULT_LOG_INTERVAL; // log int32_t tsNumOfLogLines = 10000000; int32_t mDebugFlag = 131; -int32_t sdbDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; -int32_t httpDebugFlag = 131; -int32_t mqttDebugFlag = 131; -int32_t monDebugFlag = 131; int32_t qDebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt deleted file mode 100644 index bffa415deb7cc3ebe15082051ebd22a81e45c899..0000000000000000000000000000000000000000 --- a/src/balance/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) -INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) - -ADD_LIBRARY(balance ${SRC}) diff --git a/src/balance/inc/bnInt.h b/src/balance/inc/bnInt.h deleted file mode 100644 index e924776ff151a9e81fe1254cdd1f1def04e6fef3..0000000000000000000000000000000000000000 --- a/src/balance/inc/bnInt.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BALANCE_INT_H -#define TDENGINE_BALANCE_INT_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "mnodeInt.h" -#include "mnodeDef.h" -#include "mnodeDnode.h" - -typedef struct { - int32_t size; - int32_t maxSize; - SDnodeObj **list; -} SBnDnodes; - -typedef struct { - void * timer; - bool stop; - pthread_mutex_t mutex; - pthread_cond_t cond; - pthread_t thread; -} SBnThread; - -typedef struct { - pthread_mutex_t mutex; -} SBnMgmt; - -int32_t bnInit(); -void bnCleanUp(); -bool bnStart(); -void bnCheckStatus(); -void bnCheckModules(); - -extern SBnDnodes tsBnDnodes; -extern void *tsMnodeTmr; - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/balance/inc/bnScore.h b/src/balance/inc/bnScore.h deleted file mode 100644 index a28c4459dd444001674305e404bfb0d23ed2fc3b..0000000000000000000000000000000000000000 --- a/src/balance/inc/bnScore.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BALANCE_SCORE_H -#define TDENGINE_BALANCE_SCORE_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "bnInt.h" - -void bnInitDnodes(); -void bnCleanupDnodes(); -void bnAccquireDnodes(); -void bnReleaseDnodes(); -float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h deleted file mode 100644 index 74a761299da9bb0816869dae3fd49a6afb6350bd..0000000000000000000000000000000000000000 --- a/src/balance/inc/bnThread.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BALANCE_THREAD_H -#define TDENGINE_BALANCE_THREAD_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "bnInt.h" - -int32_t bnInitThread(); -void bnCleanupThread(); -void bnNotify(); -void bnStartTimer(int32_t mseconds); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c deleted file mode 100644 index 9997d44ca55954b120ae4849d4f68be4d23419f7..0000000000000000000000000000000000000000 --- a/src/balance/src/bnMain.c +++ /dev/null @@ -1,728 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tref.h" -#include "tsync.h" -#include "tglobal.h" -#include "dnode.h" -#include "bnInt.h" -#include "bnScore.h" -#include "bnThread.h" -#include "mnodeDb.h" -#include "mnodeMnode.h" -#include "mnodeSdb.h" -#include "mnodeShow.h" -#include "mnodeUser.h" -#include "mnodeVgroup.h" - -extern int64_t tsDnodeRid; -extern int32_t tsSdbRid; -static SBnMgmt tsBnMgmt; -static void bnMonitorDnodeModule(); - -static void bnLock() { - pthread_mutex_lock(&tsBnMgmt.mutex); -} - -static void bnUnLock() { - pthread_mutex_unlock(&tsBnMgmt.mutex); -} - -static bool bnCheckFree(SDnodeObj *pDnode) { - if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) { - mError("dnode:%d, status:%s not available", pDnode->dnodeId, dnodeStatus[pDnode->status]); - return false; - } - - if (pDnode->openVnodes >= TSDB_MAX_VNODES) { - mError("dnode:%d, openVnodes:%d maxVnodes:%d not available", pDnode->dnodeId, pDnode->openVnodes, TSDB_MAX_VNODES); - return false; - } - - if (pDnode->diskAvailable <= tsMinimalDataDirGB) { - mError("dnode:%d, disk space:%fGB, not available", pDnode->dnodeId, pDnode->diskAvailable); - return false; - } - - if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_MNODE) { - mDebug("dnode:%d, alternative role is master, can't alloc vnodes in this dnode", pDnode->dnodeId); - return false; - } - - return true; -} - -static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { - SVnodeGid tmp = *pVnodeGid1; - *pVnodeGid1 = *pVnodeGid2; - *pVnodeGid2 = tmp; -} - -static void bnAdjustVnodeIndex(SVgObj *pInVg) { - int32_t d0Id = pInVg->vnodeGid[0].dnodeId; - int32_t d1Id = pInVg->vnodeGid[1].dnodeId; - int32_t d2Id = pInVg->vnodeGid[2].dnodeId; - - int32_t vgId = pInVg->vgId; - int32_t d0Num = 0; - int32_t d1Num = 0; - int32_t d2Num = 0; - - void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - - if (pVgroup->vgId != vgId) { - if (pVgroup->vnodeGid[0].dnodeId == d0Id) d0Num++; - if (pVgroup->vnodeGid[0].dnodeId == d1Id) d1Num++; - if (pVgroup->vnodeGid[0].dnodeId == d2Id) d2Num++; - } - - mnodeDecVgroupRef(pVgroup); - } - - if (pInVg->numOfVnodes == 1) { - } - - if (pInVg->numOfVnodes == 2) { - mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num); - if (d0Num > d1Num) { - mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); - } - } - - if (pInVg->numOfVnodes >= 3) { - mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num, d2Id, d2Num); - if (d0Num <= d1Num && d0Num <= d2Num) { - if (d1Num > d2Num) { - mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); - } - } else if (d1Num <= d2Num && d1Num <= d0Num) { - mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); - if (d0Num > d2Num) { - mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); - } - } else { - mDebug("vgId:%d, adjust vnode index 0 to 2", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[2]); - if (d1Num > d0Num) { - mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); - bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); - } - } - } - - for (int i = 0; i < pInVg->numOfVnodes; ++i) { - mDebug("vgId:%d index:%d dnodeId:%d", pInVg->vgId, i, pInVg->vnodeGid[i].dnodeId); - } -} - -static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { - mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId); - - SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId); - if (pDnode != NULL) { - atomic_sub_fetch_32(&pDnode->openVnodes, 1); - mnodeDecDnodeRef(pDnode); - } - - SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; memset(vnodeGid, 0, sizeof(vnodeGid)); /* = {0}; */ - int32_t numOfVnodes = 0; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pTmpVodeGid = pVgroup->vnodeGid + i; - if (pTmpVodeGid == pVnodeGid) { - continue; - } - vnodeGid[numOfVnodes] = *pTmpVodeGid; - ++numOfVnodes; - } - memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); - pVgroup->numOfVnodes = numOfVnodes; - - bnAdjustVnodeIndex(pVgroup); - mnodeUpdateVgroup(pVgroup); -} - -int32_t bnAllocVnodes(SVgObj *pVgroup) { - int32_t dnode = 0; - int32_t vnodes = 0; - - bnLock(); - bnAccquireDnodes(); - - mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes, - mnodeGetDnodesNum(), tsBnDnodes.size); - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - for (; dnode < tsBnDnodes.size; ++dnode) { - SDnodeObj *pDnode = tsBnDnodes.list[dnode]; - if (bnCheckFree(pDnode)) { - SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i; - pVnodeGid->dnodeId = pDnode->dnodeId; - pVnodeGid->pDnode = pDnode; - dnode++; - vnodes++; - mDebug("dnode:%d, is selected, vnodeIndex:%d", pDnode->dnodeId, i); - break; - } else { - mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, - dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); - } - } - } - - if (vnodes != pVgroup->numOfVnodes) { - bnReleaseDnodes(); - bnUnLock(); - - mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes); - - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - while (1) { - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, dnodeStatus[pDnode->status], - pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); - mnodeDecDnodeRef(pDnode); - } - - if (mnodeGetOnlineDnodesNum() == 0) { - return TSDB_CODE_MND_NOT_READY; - } else { - return TSDB_CODE_MND_NO_ENOUGH_DNODES; - } - } - - bnAdjustVnodeIndex(pVgroup); - bnReleaseDnodes(); - bnUnLock(); - return TSDB_CODE_SUCCESS; -} - -static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { - int32_t rmVnodeVer = 0; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pVnode = pVgroup->vnodeGid + i; - if (pVnode == pRmVnode) { - rmVnodeVer = mnodeGetVgidVer(pVnode->vver); - mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d is watching", pVgroup->vgId, i, - pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], rmVnodeVer); - } - } - - bool isReady = false; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pVnode = pVgroup->vnodeGid + i; - SDnodeObj *pDnode = pVnode->pDnode; - if (pVnode == pRmVnode) continue; - int32_t vver = mnodeGetVgidVer(pVnode->vver); - - mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d, rmvver:%d", pVgroup->vgId, i, - pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); - if (pDnode->status == TAOS_DN_STATUS_DROPPING) continue; - if (pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; - if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) continue; - - if (rmVnodeVer == 0 || vver >= rmVnodeVer) { - mInfo("vgId:%d, is ready for vindex:%d in dnode:%d status:%s role:%s vver:%d larger than rmvver:%d", - pVgroup->vgId, i, pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); - isReady = true; - } - } - - return isReady; -} - -/** - * desc: remove one vnode from vgroup - * all vnodes in vgroup should in ready state, except the balancing one - **/ -static int32_t bnRemoveVnode(SVgObj *pVgroup) { - if (pVgroup->numOfVnodes <= 1) return -1; - - SVnodeGid *pSelVnode = &pVgroup->vnodeGid[pVgroup->numOfVnodes - 1]; - mDebug("vgId:%d, vnode in dnode:%d will be dropped", pVgroup->vgId, pSelVnode->dnodeId); - - if (!bnCheckVgroupReady(pVgroup, pSelVnode)) { - mDebug("vgId:%d, is not ready", pVgroup->vgId); - return -1; - } else { - mInfo("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); - bnDiscardVnode(pVgroup, pSelVnode); - return TSDB_CODE_SUCCESS; - } -} - -static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) { - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pGid = &pVgroup->vnodeGid[i]; - if (pGid->dnodeId == 0) break; - if (pGid->dnodeId == pDnode->dnodeId) { - return true; - } - } - - return false; -} - -static SDnodeObj *bnGetAvailDnode(SVgObj *pVgroup) { - for (int32_t i = 0; i < tsBnDnodes.size; ++i) { - SDnodeObj *pDnode = tsBnDnodes.list[i]; - if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue; - if (!bnCheckFree(pDnode)) continue; - - mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId); - return pDnode; - } - - return NULL; -} - -static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { - if (pDestDnode == NULL || pSrcDnode == pDestDnode) { - return TSDB_CODE_MND_DNODE_NOT_EXIST; - } - - SVnodeGid vnodeGids[TSDB_MAX_REPLICA]; - memcpy(&vnodeGids, &pVgroup->vnodeGid, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); - - int32_t numOfVnodes = pVgroup->numOfVnodes; - vnodeGids[numOfVnodes].dnodeId = pDestDnode->dnodeId; - vnodeGids[numOfVnodes].pDnode = pDestDnode; - numOfVnodes++; - - // move the src vnode to the end - for (int32_t v = 0; v < numOfVnodes; ++v) { - if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) { - bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]); - pVgroup->lbDnodeId = pSrcDnode->dnodeId; - break; - } - } - - // adjust the vgroup postion - if (pSrcDnode == NULL) { - bnAdjustVnodeIndex(pVgroup); - } - - memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); - pVgroup->numOfVnodes = numOfVnodes; - atomic_add_fetch_32(&pDestDnode->openVnodes, 1); - - mnodeUpdateVgroup(pVgroup); - - return TSDB_CODE_SUCCESS; -} - -static bool bnMonitorBalance() { - if (tsBnDnodes.size < 2) return false; - - mDebug("monitor dnodes for balance, avail:%d", tsBnDnodes.size); - for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) { - SDnodeObj *pDnode = tsBnDnodes.list[src]; - mDebug("%d-dnode:%d, state:%s, score:%.1f, cores:%d, vnodes:%d", tsBnDnodes.size - src - 1, pDnode->dnodeId, - dnodeStatus[pDnode->status], pDnode->score, pDnode->numOfCores, pDnode->openVnodes); - } - - float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score; - if (scoresDiff < 0.01) { - mDebug("all dnodes:%d is already balanced, scoreDiff:%.1f", tsBnDnodes.size, scoresDiff); - return false; - } - - for (int32_t src = tsBnDnodes.size - 1; src > 0; --src) { - SDnodeObj *pSrcDnode = tsBnDnodes.list[src]; - float srcScore = bnTryCalcDnodeScore(pSrcDnode, -1); - if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) { - continue; - } - - void *pIter = NULL; - while (1) { - SVgObj *pVgroup; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - - if (bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) { - for (int32_t dest = 0; dest < src; dest++) { - SDnodeObj *pDestDnode = tsBnDnodes.list[dest]; - if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) continue; - if (taosGetTimestampMs() - pDestDnode->createdTime < 2000) continue; - - float destScore = bnTryCalcDnodeScore(pDestDnode, 1); - if (srcScore + 0.0001 < destScore) continue; - if (!bnCheckFree(pDestDnode)) continue; - - mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f", - pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score, - srcScore, pDestDnode->score, destScore); - bnAddVnode(pVgroup, pSrcDnode, pDestDnode); - mnodeDecVgroupRef(pVgroup); - mnodeCancelGetNextVgroup(pIter); - return true; - } - } - - mnodeDecVgroupRef(pVgroup); - } - } - - return false; -} - -// if mgmt changed to master -// 1. reset balanceAccessSquence to zero -// 2. reset state of dnodes to offline -// 3. reset lastAccess of dnodes to zero -void bnReset() { - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - while (1) { - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - - // while master change, should reset dnode to offline - mInfo("dnode:%d set access:%" PRId64 " to 0", pDnode->dnodeId, pDnode->lastAccess); - pDnode->lastAccess = 0; - if (pDnode->status != TAOS_DN_STATUS_DROPPING) { - pDnode->status = TAOS_DN_STATUS_OFFLINE; - pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; - } - - mnodeDecDnodeRef(pDnode); - } - - tsAccessSquence = 0; -} - -static bool bnMonitorVgroups() { - void * pIter = NULL; - SVgObj *pVgroup = NULL; - bool hasUpdatingVgroup = false; - - while (1) { - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL || pVgroup->pDb == NULL) break; - - int32_t dbReplica = pVgroup->pDb->cfg.replications; - int32_t vgReplica = pVgroup->numOfVnodes; - int32_t code = -1; - - if (vgReplica > dbReplica) { - mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica); - hasUpdatingVgroup = true; - code = bnRemoveVnode(pVgroup); - } else if (vgReplica < dbReplica) { - mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica); - hasUpdatingVgroup = true; - - SDnodeObj *pAvailDnode = bnGetAvailDnode(pVgroup); - if (pAvailDnode == NULL) { - code = TSDB_CODE_MND_DNODE_NOT_EXIST; - } else { - code = bnAddVnode(pVgroup, NULL, pAvailDnode); - } - } - - mnodeDecVgroupRef(pVgroup); - if (code == TSDB_CODE_SUCCESS) { - mnodeCancelGetNextVgroup(pIter); - break; - } - } - - return hasUpdatingVgroup; -} - -static bool bnMonitorDnodeDropping(SDnodeObj *pDnode) { - mDebug("dnode:%d, in dropping state", pDnode->dnodeId); - - void * pIter = NULL; - bool hasThisDnode = false; - while (1) { - SVgObj *pVgroup = NULL; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - - hasThisDnode = bnCheckDnodeInVgroup(pDnode, pVgroup); - mnodeDecVgroupRef(pVgroup); - - if (hasThisDnode) { - mnodeCancelGetNextVgroup(pIter); - break; - } - } - - if (!hasThisDnode) { - mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId); - mnodeDropDnode(pDnode, NULL); - return true; - } - - return false; -} - -static bool bnMontiorDropping() { - void *pIter = NULL; - SDnodeObj *pDnode = NULL; - - while (1) { - mnodeDecDnodeRef(pDnode); - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - - if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { - if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue; - if (dnodeIsMasterEp(pDnode->dnodeEp)) continue; - if (mnodeGetDnodesNum() <= 1) continue; - - mLInfo("dnode:%d, set to removing state for it offline:%" PRId64 " seconds", pDnode->dnodeId, - tsAccessSquence - pDnode->lastAccess); - - pDnode->status = TAOS_DN_STATUS_DROPPING; - mnodeUpdateDnode(pDnode); - mnodeDecDnodeRef(pDnode); - mnodeCancelGetNextDnode(pIter); - return true; - } - - if (pDnode->status == TAOS_DN_STATUS_DROPPING) { - bool ret = bnMonitorDnodeDropping(pDnode); - mnodeDecDnodeRef(pDnode); - mnodeCancelGetNextDnode(pIter); - return ret; - } - } - - return false; -} - -bool bnStart() { - if (!sdbIsMaster()) return false; - - bnLock(); - bnAccquireDnodes(); - - bnMonitorDnodeModule(); - - bool updateSoon = bnMontiorDropping(); - - if (!updateSoon) { - updateSoon = bnMonitorVgroups(); - } - - if (!updateSoon) { - updateSoon = bnMonitorBalance(); - } - - bnReleaseDnodes(); - bnUnLock(); - - return updateSoon; -} - -static void bnSetVgroupOffline(SDnodeObj* pDnode) { - void *pIter = NULL; - while (1) { - SVgObj *pVgroup; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - if (pVgroup->vnodeGid[i].pDnode == pDnode) { - pVgroup->vnodeGid[i].role = TAOS_SYNC_ROLE_OFFLINE; - } - } - mnodeDecVgroupRef(pVgroup); - } -} - -void bnCheckStatus() { - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - - void *dnodeSdb = taosAcquireRef(tsSdbRid, tsDnodeRid); - if (dnodeSdb == NULL) return; - - while (1) { - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - if (tsAccessSquence - pDnode->lastAccess > 3) { - if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) { - pDnode->status = TAOS_DN_STATUS_OFFLINE; - pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT; - mInfo("dnode:%d, set to offline state, access seq:%" PRId64 " last seq:%" PRId64 " laststat:%d", pDnode->dnodeId, - tsAccessSquence, pDnode->lastAccess, pDnode->status); - bnSetVgroupOffline(pDnode); - bnStartTimer(3000); - } - } - mnodeDecDnodeRef(pDnode); - } - - taosReleaseRef(tsSdbRid, tsDnodeRid); -} - -void bnCheckModules() { - if (sdbIsMaster()) { - bnLock(); - bnAccquireDnodes(); - bnMonitorDnodeModule(); - bnReleaseDnodes(); - bnUnLock(); - } -} - -int32_t bnInit() { - pthread_mutex_init(&tsBnMgmt.mutex, NULL); - bnInitDnodes(); - bnInitThread(); - bnReset(); - - return 0; -} - -void bnCleanUp() { - bnCleanupThread(); - bnCleanupDnodes(); - pthread_mutex_destroy(&tsBnMgmt.mutex); -} - -int32_t bnDropDnode(SDnodeObj *pDnode) { - int32_t totalFreeVnodes = 0; - void * pIter = NULL; - SDnodeObj *pTempDnode = NULL; - - while (1) { - pIter = mnodeGetNextDnode(pIter, &pTempDnode); - if (pTempDnode == NULL) break; - - if (pTempDnode != pDnode && bnCheckFree(pTempDnode)) { - totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes); - } - - mnodeDecDnodeRef(pTempDnode); - } - - if (pDnode->openVnodes > totalFreeVnodes) { - mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes); - return TSDB_CODE_MND_NO_ENOUGH_DNODES; - } - - pDnode->status = TAOS_DN_STATUS_DROPPING; - mnodeUpdateDnode(pDnode); - - bnStartTimer(1100); - - return TSDB_CODE_SUCCESS; -} - -int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode) { - if (pDnode == NULL) - return 0; - - if (pDnode->isMgmt || pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE - || pDnode->status == TAOS_DN_STATUS_DROPPING - || pDnode->status == TAOS_DN_STATUS_OFFLINE) { - return 0; - } else { - return 1; - } -} - -static void bnMonitorDnodeModule() { - int32_t numOfMnodes = mnodeGetMnodesNum(); - if (numOfMnodes >= tsNumOfMnodes) return; - - for (int32_t i = 0; i < tsBnDnodes.size; ++i) { - SDnodeObj *pDnode = tsBnDnodes.list[i]; - if (pDnode == NULL) break; - - if (!bnDnodeCanCreateMnode(pDnode)) continue; - - mLInfo("dnode:%d, numOfMnodes:%d expect:%d, create mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes); - mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, true); - -#if 0 - // Only create one mnode each time - return; -#else - numOfMnodes = mnodeGetMnodesNum(); - if (numOfMnodes >= tsNumOfMnodes) return; -#endif - } -} - -int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) { - if (!sdbIsMaster()) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId); - return TSDB_CODE_MND_DNODE_NOT_EXIST; - } - - if (tsEnableBalance != 0) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for balance enabled", pSrcDnode->dnodeId, vnodeId, dnodeId); - return TSDB_CODE_MND_BALANCE_ENABLED; - } - - SVgObj *pVgroup = mnodeGetVgroup(vnodeId); - if (pVgroup == NULL) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for vgroup not exist", pSrcDnode->dnodeId, vnodeId, dnodeId); - return TSDB_CODE_MND_VGROUP_NOT_EXIST; - } - - SDnodeObj *pDestDnode = mnodeGetDnode(dnodeId); - if (pDestDnode == NULL) { - mnodeDecVgroupRef(pVgroup); - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode not exist", pSrcDnode->dnodeId, vnodeId, dnodeId); - return TSDB_CODE_MND_DNODE_NOT_EXIST; - } - - bnLock(); - bnAccquireDnodes(); - - int32_t code = TSDB_CODE_SUCCESS; - if (!bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId, - dnodeId, pSrcDnode->dnodeId); - code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE; - } else if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId, - dnodeId, dnodeId); - code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE; - } else if (!bnCheckFree(pDestDnode)) { - mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId, - dnodeId); - code = TSDB_CODE_MND_DNODE_NOT_FREE; - } else { - code = bnAddVnode(pVgroup, pSrcDnode, pDestDnode); - mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code)); - } - - bnReleaseDnodes(); - bnUnLock(); - - mnodeDecVgroupRef(pVgroup); - mnodeDecDnodeRef(pDestDnode); - - return code; -} diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c deleted file mode 100644 index 04a14357c9e602807f5aa254d8a5ea25bc4b328d..0000000000000000000000000000000000000000 --- a/src/balance/src/bnScore.c +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "mnodeShow.h" -#include "mnodeUser.h" -#include "bnScore.h" - -SBnDnodes tsBnDnodes; - -static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn); - -static int32_t bnCalcCpuScore(SDnodeObj *pDnode) { - if (pDnode->cpuAvgUsage < 80) - return 0; - else if (pDnode->cpuAvgUsage < 90) - return 10; - else - return 50; -} - -static int32_t bnCalcMemoryScore(SDnodeObj *pDnode) { - if (pDnode->memoryAvgUsage < 80) - return 0; - else if (pDnode->memoryAvgUsage < 90) - return 10; - else - return 50; -} - -static int32_t bnCalcDiskScore(SDnodeObj *pDnode) { - if (pDnode->diskAvgUsage < 80) - return 0; - else if (pDnode->diskAvgUsage < 90) - return 10; - else - return 50; -} - -static int32_t bnCalcBandScore(SDnodeObj *pDnode) { - if (pDnode->bandwidthUsage < 30) - return 0; - else if (pDnode->bandwidthUsage < 80) - return 10; - else - return 50; -} - -static float bnCalcModuleScore(SDnodeObj *pDnode) { - if (pDnode->numOfCores <= 0) return 0; - if (pDnode->isMgmt) { - return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores; - } - return 0; -} - -static float bnCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) { - if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000; - if (pDnode->numOfCores <= 0) return 0; - return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores; -} - -/** - * calc singe score, such as cpu/memory/disk/bandwitdh/vnode - * 1. get the score config - * 2. if the value is out of range, use border data - * 3. otherwise use interpolation method - **/ -static void bnCalcDnodeScore(SDnodeObj *pDnode) { - pDnode->score = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + - bnCalcBandScore(pDnode) + bnCalcModuleScore(pDnode) + bnCalcVnodeScore(pDnode, 0) + - pDnode->customScore; -} - -float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) { - int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + - bnCalcBandScore(pDnode); - float moduleScore = bnCalcModuleScore(pDnode); - float vnodeScore = bnCalcVnodeScore(pDnode, extra); - - float score = systemScore + moduleScore + vnodeScore + pDnode->customScore; - return score; -} - -void bnInitDnodes() { - mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, bnGetScoresMeta); - mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, bnRetrieveScores); - mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode); - - memset(&tsBnDnodes, 0, sizeof(SBnDnodes)); - tsBnDnodes.maxSize = 16; - tsBnDnodes.list = calloc(tsBnDnodes.maxSize, sizeof(SDnodeObj *)); -} - -void bnCleanupDnodes() { - if (tsBnDnodes.list != NULL) { - free(tsBnDnodes.list); - tsBnDnodes.list = NULL; - } -} - -static void bnCheckDnodesSize(int32_t dnodesNum) { - if (tsBnDnodes.maxSize <= dnodesNum) { - int32_t maxSize = dnodesNum * 2; - SDnodeObj** list1 = NULL; - int32_t retry = 0; - - while(list1 == NULL && retry++ < 3) { - list1 = realloc(tsBnDnodes.list, maxSize * sizeof(SDnodeObj *)); - } - if(list1) { - tsBnDnodes.list = list1; - tsBnDnodes.maxSize = maxSize; - } - } -} - -void bnAccquireDnodes() { - int32_t dnodesNum = mnodeGetDnodesNum(); - bnCheckDnodesSize(dnodesNum); - - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - int32_t dnodeIndex = 0; - - while (1) { - if (dnodeIndex >= dnodesNum) { - mnodeCancelGetNextDnode(pIter); - break; - } - - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { - mnodeDecDnodeRef(pDnode); - continue; - } - - bnCalcDnodeScore(pDnode); - - int32_t orderIndex = dnodeIndex; - for (; orderIndex > 0; --orderIndex) { - if (pDnode->score > tsBnDnodes.list[orderIndex - 1]->score) { - break; - } - tsBnDnodes.list[orderIndex] = tsBnDnodes.list[orderIndex - 1]; - } - tsBnDnodes.list[orderIndex] = pDnode; - dnodeIndex++; - } - - tsBnDnodes.size = dnodeIndex; -} - -void bnReleaseDnodes() { - for (int32_t i = 0; i < tsBnDnodes.size; ++i) { - SDnodeObj *pDnode = tsBnDnodes.list[i]; - if (pDnode != NULL) { - mnodeDecDnodeRef(pDnode); - } - } -} - -static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { - SUserObj *pUser = mnodeGetUserFromConn(pConn); - if (pUser == NULL) return 0; - - if (strcmp(pUser->pAcct->user, "root") != 0) { - mnodeDecUserRef(pUser); - return TSDB_CODE_MND_NO_RIGHTS; - } - - int32_t cols = 0; - SSchema *pSchema = pMeta->schema; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "system scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "custom scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "module scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "vnode scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "total scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "open vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cpu cores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "balance state"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htons(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) { - pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - } - - pShow->numOfRows = mnodeGetDnodesNum(); - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pIter = NULL; - - mnodeDecUserRef(pUser); - - return 0; -} - -static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) { - int32_t numOfRows = 0; - SDnodeObj *pDnode = NULL; - char * pWrite; - int32_t cols = 0; - - while (numOfRows < rows) { - pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode); - if (pDnode == NULL) break; - - int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + bnCalcBandScore(pDnode); - float moduleScore = bnCalcModuleScore(pDnode); - float vnodeScore = bnCalcVnodeScore(pDnode, 0); - - cols = 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDnode->dnodeId; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)systemScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)pDnode->customScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)moduleScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)vnodeScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (float)(vnodeScore + moduleScore + pDnode->customScore + systemScore); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDnode->openVnodes; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDnode->numOfCores; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, dnodeStatus[pDnode->status]); - cols++; - - numOfRows++; - mnodeDecDnodeRef(pDnode); - } - - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); - pShow->numOfReads += numOfRows; - return numOfRows; -} diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c deleted file mode 100644 index 20da83ccba4c192a733cdbb530a2b6aab358896a..0000000000000000000000000000000000000000 --- a/src/balance/src/bnThread.c +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ttimer.h" -#include "tglobal.h" -#include "mnodeSdb.h" -#include "bnThread.h" - -static SBnThread tsBnThread; - -static void *bnThreadFunc(void *arg) { - setThreadName("balance"); - - while (1) { - pthread_mutex_lock(&tsBnThread.mutex); - if (tsBnThread.stop) { - pthread_mutex_unlock(&tsBnThread.mutex); - break; - } - - pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex); - mDebug("balance thread wakes up to work"); - bool updateSoon = bnStart(); - mDebug("balance thread finished this poll, updateSoon:%d", updateSoon); - - bnStartTimer(updateSoon ? 1000 : -1); - pthread_mutex_unlock(&(tsBnThread.mutex)); - } - - mDebug("balance thread is stopped"); - return NULL; -} - -int32_t bnInitThread() { - memset(&tsBnThread, 0, sizeof(SBnThread)); - tsBnThread.stop = false; - pthread_mutex_init(&tsBnThread.mutex, NULL); - pthread_cond_init(&tsBnThread.cond, NULL); - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL); - pthread_attr_destroy(&thattr); - - if (ret != 0) { - mError("failed to create balance thread since %s", strerror(ret)); - return -1; - } - - bnStartTimer(2000); - mDebug("balance thread is created"); - return 0; -} - -void bnCleanupThread() { - mDebug("balance thread will be cleanup"); - - if (tsBnThread.timer != NULL) { - taosTmrStopA(&tsBnThread.timer); - tsBnThread.timer = NULL; - mDebug("stop balance timer"); - } - - pthread_mutex_lock(&tsBnThread.mutex); - tsBnThread.stop = true; - pthread_cond_signal(&tsBnThread.cond); - pthread_mutex_unlock(&(tsBnThread.mutex)); - pthread_join(tsBnThread.thread, NULL); - - pthread_cond_destroy(&tsBnThread.cond); - pthread_mutex_destroy(&tsBnThread.mutex); -} - -static void bnPostSignal() { - if (tsBnThread.stop) return; - - pthread_mutex_lock(&tsBnThread.mutex); - pthread_cond_signal(&tsBnThread.cond); - pthread_mutex_unlock(&(tsBnThread.mutex)); -} - -/* - * once sdb work as mater, then tsAccessSquence reset to zero - * increase tsAccessSquence every balance interval - */ - -static void bnProcessTimer(void *handle, void *tmrId) { - if (!sdbIsMaster()) return; - if (tsBnThread.stop) return; - - tsBnThread.timer = NULL; - bnStartTimer(-1); - bnCheckStatus(); - - if (handle == NULL) { - ++tsAccessSquence; - - if (tsAccessSquence % tsBalanceInterval == 0) { - mDebug("balance function is scheduled by timer"); - bnPostSignal(); - } - } else { - int64_t mseconds = (int64_t)handle; - mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds); - bnPostSignal(); - } -} - -void bnStartTimer(int32_t mseconds) { - if (tsBnThread.stop) return; - - if (mseconds != -1) { - mTrace("balance function will be called after %d ms", mseconds); - taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer); - } else { - taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); - } -} - -void bnNotify() { - bnStartTimer(500); -} diff --git a/src/inc/http.h b/src/inc/http.h deleted file mode 100644 index 0d4c386cbf1e784834019c5d75847ab20b7ce8e9..0000000000000000000000000000000000000000 --- a/src/inc/http.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_H -#define TDENGINE_HTTP_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -int32_t httpGetReqCount(); -int32_t httpInitSystem(); -int32_t httpStartSystem(); -void httpStopSystem(); -void httpCleanUpSystem(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/monitor.h b/src/inc/monitor.h deleted file mode 100644 index d2e5e06487dbdf311cef6da125d7ba3050b53a4d..0000000000000000000000000000000000000000 --- a/src/inc/monitor.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MONITOR_H -#define TDENGINE_MONITOR_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -typedef struct { - char * acctId; - int64_t currentPointsPerSecond; - int64_t maxPointsPerSecond; - int64_t totalTimeSeries; - int64_t maxTimeSeries; - int64_t totalStorage; - int64_t maxStorage; - int64_t totalQueryTime; - int64_t maxQueryTime; - int64_t totalInbound; - int64_t maxInbound; - int64_t totalOutbound; - int64_t maxOutbound; - int64_t totalDbs; - int64_t maxDbs; - int64_t totalUsers; - int64_t maxUsers; - int64_t totalStreams; - int64_t maxStreams; - int64_t totalConns; - int64_t maxConns; - int8_t accessState; -} SAcctMonitorObj; - -int32_t monInitSystem(); -int32_t monStartSystem(); -void monStopSystem(); -void monCleanupSystem(); -void monSaveAcctLog(SAcctMonitorObj *pMonObj); -void monSaveLog(int32_t level, const char *const format, ...); -void monExecuteSQL(char *sql); -typedef void (*MonExecuteSQLCbFP)(void *param, TAOS_RES *, int code); -void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param); -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tacct.h b/src/inc/tacct.h deleted file mode 100644 index 52215fac526f91355efafaf9edb2fb1366dd46fb..0000000000000000000000000000000000000000 --- a/src/inc/tacct.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_ACCT_H -#define TDENGINE_ACCT_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum { - ACCT_GRANT_USER, - ACCT_GRANT_DB, - ACCT_GRANT_TABLE -} EAcctGrantType; - -int32_t acctInit(); -void acctCleanUp(); -int32_t acctCheck(void *pAcct, EAcctGrantType type); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tadmin.h b/src/inc/tadmin.h deleted file mode 100644 index b7de33576a3e73ea481bb75d01f1005b3bd942f6..0000000000000000000000000000000000000000 --- a/src/inc/tadmin.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_ADMIN_H -#define TDENGINE_ADMIN_H - -#ifdef __cplusplus -extern "C" { -#endif - -struct HttpServer; - -void adminInitHandle(struct HttpServer* pServer); -void opInitHandle(struct HttpServer* pServer); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tbn.h b/src/inc/tbn.h deleted file mode 100644 index b35f90eb153d02b4bb09f6f96a8dd09835626c97..0000000000000000000000000000000000000000 --- a/src/inc/tbn.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BALANCE_H -#define TDENGINE_BALANCE_H - -#ifdef __cplusplus -extern "C" { -#endif - -struct SVgObj; -struct SDnodeObj; - -int32_t bnInit(); -void bnCleanUp(); -void bnNotify(); -void bnCheckModules(); -void bnReset(); -int32_t bnAllocVnodes(struct SVgObj *pVgroup); -int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId); -int32_t bnDropDnode(struct SDnodeObj *pDnode); -int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tgrant.h b/src/inc/tgrant.h deleted file mode 100644 index f62a521b6cd157074ab18a4bf1a1a18f9120ac97..0000000000000000000000000000000000000000 --- a/src/inc/tgrant.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_GTANT_H -#define TDENGINE_GTANT_H - -#ifdef __cplusplus -"C" { -#endif - -typedef enum { - TSDB_GRANT_ALL, - TSDB_GRANT_TIME, - TSDB_GRANT_USER, - TSDB_GRANT_DB, - TSDB_GRANT_TIMESERIES, - TSDB_GRANT_DNODE, - TSDB_GRANT_ACCT, - TSDB_GRANT_STORAGE, - TSDB_GRANT_SPEED, - TSDB_GRANT_QUERY_TIME, - TSDB_GRANT_CONNS, - TSDB_GRANT_STREAMS, - TSDB_GRANT_CPU_CORES, -} EGrantType; - -int32_t grantInit(); -void grantCleanUp(); -void grantParseParameter(); -int32_t grantCheck(EGrantType grant); -void grantReset(EGrantType grant, uint64_t value); -void grantAdd(EGrantType grant, uint64_t value); -void grantRestore(EGrantType grant, uint64_t value); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/tsync.h b/src/inc/tsync.h deleted file mode 100644 index d1b68e3f5a1e27b63dd08a4d1d4862c7b1e68179..0000000000000000000000000000000000000000 --- a/src/inc/tsync.h +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_H -#define TDENGINE_SYNC_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TAOS_SYNC_MAX_REPLICA 5 -#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF - -typedef enum { - TAOS_SYNC_ROLE_OFFLINE = 0, - TAOS_SYNC_ROLE_UNSYNCED = 1, - TAOS_SYNC_ROLE_SYNCING = 2, - TAOS_SYNC_ROLE_SLAVE = 3, - TAOS_SYNC_ROLE_MASTER = 4 -} ESyncRole; - -typedef enum { - TAOS_SYNC_STATUS_INIT = 0, - TAOS_SYNC_STATUS_START = 1, - TAOS_SYNC_STATUS_FILE = 2, - TAOS_SYNC_STATUS_CACHE = 3 -} ESyncStatus; - -typedef struct { - uint32_t nodeId; // node ID assigned by TDengine - uint16_t nodePort; // node sync Port - char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN -} SNodeInfo; - -typedef struct { - int8_t quorum; // number of confirms required, >=1 - int8_t replica; // number of replications, >=1 - SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; -} SSyncCfg; - -typedef struct { - int32_t selfIndex; - uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; - int32_t role[TAOS_SYNC_MAX_REPLICA]; -} SNodesRole; - -// get the wal file from index or after -// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file -typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); - -// when a forward pkt is received, call this to handle data -typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); - -// when forward is confirmed by peer, master call this API to notify app -typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); - -// when role is changed, call this to notify app -typedef void (*FNotifyRole)(int32_t vgId, int8_t role); - -// if a number of retrieving data failed, call this to start flow control -typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); - -// when data file is synced successfully, notity app -typedef void (*FStartSyncFile)(int32_t vgId); -typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); - -// get file version -typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); - -typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); -typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); - -typedef struct { - int32_t vgId; // vgroup ID - uint64_t version; // initial version - SSyncCfg syncCfg; // configuration from mgmt - char path[TSDB_FILENAME_LEN]; // path to the file - void * pTsdb; - FGetWalInfo getWalInfoFp; - FWriteToCache writeToCacheFp; - FConfirmForward confirmForward; - FNotifyRole notifyRoleFp; - FNotifyFlowCtrl notifyFlowCtrlFp; - FStartSyncFile startSyncFileFp; - FStopSyncFile stopSyncFileFp; - FGetVersion getVersionFp; - FSendFile sendFileFp; - FRecvFile recvFileFp; -} SSyncInfo; - -typedef void *tsync_h; - -int32_t syncInit(); -void syncCleanUp(); - -int64_t syncStart(const SSyncInfo *); -void syncStop(int64_t rid); -int32_t syncReconfig(int64_t rid, const SSyncCfg *); -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force); -void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force); -void syncRecover(int64_t rid); // recover from other nodes: -int32_t syncGetNodesRole(int64_t rid, SNodesRole *); - -extern char *syncRole[]; - -//global configurable parameters -extern int32_t sDebugFlag; -extern char tsArbitrator[]; -extern uint16_t tsSyncPort; - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SYNC_H diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt deleted file mode 100644 index 8a05d63e141facfe34e740887384fec0337534d4..0000000000000000000000000000000000000000 --- a/src/plugins/monitor/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -AUX_SOURCE_DIRECTORY(./src SRC) - -ADD_LIBRARY(monitor ${SRC}) - -IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(monitor taos_static) -ELSE () - TARGET_LINK_LIBRARIES(monitor taos) -ENDIF () diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c deleted file mode 100644 index fea793fa860fd17ff30bcecae1436180bc6b34bf..0000000000000000000000000000000000000000 --- a/src/plugins/monitor/src/monMain.c +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tlog.h" -#include "ttimer.h" -#include "tutil.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "dnode.h" -#include "monitor.h" -#include "taoserror.h" - -#define monFatal(...) { if (monDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} -#define monError(...) { if (monDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} -#define monWarn(...) { if (monDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} -#define monInfo(...) { if (monDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }} -#define monDebug(...) { if (monDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }} -#define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }} - -#define SQL_LENGTH 1030 -#define LOG_LEN_STR 512 -#define IP_LEN_STR TSDB_EP_LEN -#define CHECK_INTERVAL 1000 - -typedef enum { - MON_CMD_CREATE_DB, - MON_CMD_CREATE_TB_LOG, - MON_CMD_CREATE_MT_DN, - MON_CMD_CREATE_MT_ACCT, - MON_CMD_CREATE_TB_DN, - MON_CMD_CREATE_TB_ACCT_ROOT, - MON_CMD_CREATE_TB_SLOWQUERY, - MON_CMD_MAX -} EMonCmd; - -typedef enum { - MON_STATE_NOT_INIT, - MON_STATE_INITED -} EMonState; - -typedef struct { - pthread_t thread; - void * conn; - char ep[TSDB_EP_LEN]; - int8_t cmdIndex; - int8_t state; - int8_t start; // enable/disable by mnode - int8_t quiting; // taosd is quiting - char sql[SQL_LENGTH + 1]; -} SMonConn; - -static SMonConn tsMonitor = {0}; -static void monSaveSystemInfo(); -static void *monThreadFunc(void *param); -static void monBuildMonitorSql(char *sql, int32_t cmd); -extern int32_t (*monStartSystemFp)(); -extern void (*monStopSystemFp)(); -extern void (*monExecuteSQLFp)(char *sql); - -int32_t monInitSystem() { - if (tsMonitor.ep[0] == 0) { - strcpy(tsMonitor.ep, tsLocalEp); - } - - int32_t len = (int32_t)strlen(tsMonitor.ep); - for (int32_t i = 0; i < len; ++i) { - if (tsMonitor.ep[i] == ':' || tsMonitor.ep[i] == '-' || tsMonitor.ep[i] == '.') { - tsMonitor.ep[i] = '_'; - } - } - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&tsMonitor.thread, &thAttr, monThreadFunc, NULL)) { - monError("failed to create thread to for monitor module, reason:%s", strerror(errno)); - return -1; - } - - pthread_attr_destroy(&thAttr); - monDebug("monitor thread is launched"); - - monStartSystemFp = monStartSystem; - monStopSystemFp = monStopSystem; - return 0; -} - -int32_t monStartSystem() { - if (taos_init()) { - return -1; - } - tsMonitor.start = 1; - monExecuteSQLFp = monExecuteSQL; - monInfo("monitor module start"); - return 0; -} - -static void *monThreadFunc(void *param) { - monDebug("starting to initialize monitor module ..."); - setThreadName("monitor"); - - while (1) { - static int32_t accessTimes = 0; - accessTimes++; - taosMsleep(1000); - - if (tsMonitor.quiting) { - tsMonitor.state = MON_STATE_NOT_INIT; - monInfo("monitor thread will quit, for taosd is quiting"); - break; - } else { - taosGetDisk(); - } - - if (tsMonitor.start == 0) { - continue; - } - - if (dnodeGetDnodeId() <= 0) { - monDebug("dnode not initialized, waiting for 3000 ms to start monitor module"); - continue; - } - - if (tsMonitor.conn == NULL) { - tsMonitor.state = MON_STATE_NOT_INIT; - tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0); - if (tsMonitor.conn == NULL) { - monError("failed to connect to database, reason:%s", tstrerror(terrno)); - continue; - } else { - monDebug("connect to database success"); - } - } - - if (tsMonitor.state == MON_STATE_NOT_INIT) { - int32_t code = 0; - - for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) { - monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex); - void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - code = taos_errno(res); - taos_free_result(res); - - if (code != 0) { - monError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code)); - break; - } else { - monDebug("successfully to exec sql:%s", tsMonitor.sql); - } - } - - if (tsMonitor.start && code == 0) { - tsMonitor.state = MON_STATE_INITED; - } - } - - if (tsMonitor.state == MON_STATE_INITED) { - if (accessTimes % tsMonitorInterval == 0) { - monSaveSystemInfo(); - } - } - } - - monInfo("monitor thread is stopped"); - return NULL; -} - -static void monBuildMonitorSql(char *sql, int32_t cmd) { - memset(sql, 0, SQL_LENGTH); - -#ifdef _STORAGE - char *keepValue = "30,30,30"; -#else - char *keepValue = "30"; -#endif - - if (cmd == MON_CMD_CREATE_DB) { - snprintf(sql, SQL_LENGTH, - "create database if not exists %s replica 1 days 10 keep %s cache %d " - "blocks %d precision 'us'", - tsMonitorDbName, keepValue, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MIN_TOTAL_BLOCKS); - } else if (cmd == MON_CMD_CREATE_MT_DN) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.dn(ts timestamp" - ", cpu_taosd float, cpu_system float, cpu_cores int" - ", mem_taosd float, mem_system float, mem_total int" - ", disk_used float, disk_total int" - ", band_speed float" - ", io_read float, io_write float" - ", req_http int, req_select int, req_insert int" - ") tags (dnodeid int, fqdn binary(%d))", - tsMonitorDbName, TSDB_FQDN_LEN); - } else if (cmd == MON_CMD_CREATE_TB_DN) { - snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn%d using %s.dn tags(%d, '%s')", tsMonitorDbName, - dnodeGetDnodeId(), tsMonitorDbName, dnodeGetDnodeId(), tsLocalEp); - } else if (cmd == MON_CMD_CREATE_MT_ACCT) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.acct(ts timestamp " - ", currentPointsPerSecond bigint, maxPointsPerSecond bigint" - ", totalTimeSeries bigint, maxTimeSeries bigint" - ", totalStorage bigint, maxStorage bigint" - ", totalQueryTime bigint, maxQueryTime bigint" - ", totalInbound bigint, maxInbound bigint" - ", totalOutbound bigint, maxOutbound bigint" - ", totalDbs smallint, maxDbs smallint" - ", totalUsers smallint, maxUsers smallint" - ", totalStreams smallint, maxStreams smallint" - ", totalConns smallint, maxConns smallint" - ", accessState smallint" - ") tags (acctId binary(%d))", - tsMonitorDbName, TSDB_USER_LEN); - } else if (cmd == MON_CMD_CREATE_TB_ACCT_ROOT) { - snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, TSDB_DEFAULT_USER, - tsMonitorDbName, TSDB_DEFAULT_USER); - } else if (cmd == MON_CMD_CREATE_TB_SLOWQUERY) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.slowquery(ts timestamp, username " - "binary(%d), created_time timestamp, time bigint, sql binary(%d))", - tsMonitorDbName, TSDB_TABLE_FNAME_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN); - } else if (cmd == MON_CMD_CREATE_TB_LOG) { - snprintf(sql, SQL_LENGTH, - "create table if not exists %s.log(ts timestamp, level tinyint, " - "content binary(%d), ipaddr binary(%d))", - tsMonitorDbName, LOG_LEN_STR, IP_LEN_STR); - } - - sql[SQL_LENGTH] = 0; -} - -void monStopSystem() { - tsMonitor.start = 0; - tsMonitor.state = MON_STATE_NOT_INIT; - monExecuteSQLFp = NULL; - monInfo("monitor module stopped"); -} - -void monCleanupSystem() { - tsMonitor.quiting = 1; - monStopSystem(); - if (taosCheckPthreadValid(tsMonitor.thread)) { - pthread_join(tsMonitor.thread, NULL); - } - - if (tsMonitor.conn != NULL) { - taos_close(tsMonitor.conn); - tsMonitor.conn = NULL; - } - monInfo("monitor module is cleaned up"); -} - -// unit is MB -static int32_t monBuildMemorySql(char *sql) { - float sysMemoryUsedMB = 0; - bool suc = taosGetSysMemory(&sysMemoryUsedMB); - if (!suc) { - monDebug("failed to get sys memory info"); - } - - float procMemoryUsedMB = 0; - suc = taosGetProcMemory(&procMemoryUsedMB); - if (!suc) { - monDebug("failed to get proc memory info"); - } - - return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB); -} - -// unit is % -static int32_t monBuildCpuSql(char *sql) { - float sysCpuUsage = 0, procCpuUsage = 0; - bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); - if (!suc) { - monDebug("failed to get cpu usage"); - } - - if (sysCpuUsage <= procCpuUsage) { - sysCpuUsage = procCpuUsage + 0.1f; - } - - return sprintf(sql, ", %f, %f, %d", procCpuUsage, sysCpuUsage, tsNumOfCores); -} - -// unit is GB -static int32_t monBuildDiskSql(char *sql) { - return sprintf(sql, ", %f, %d", tsUsedDataDirGB, (int32_t)tsTotalDataDirGB); -} - -// unit is Kb -static int32_t monBuildBandSql(char *sql) { - float bandSpeedKb = 0; - bool suc = taosGetBandSpeed(&bandSpeedKb); - if (!suc) { - monDebug("failed to get bandwidth speed"); - } - - return sprintf(sql, ", %f", bandSpeedKb); -} - -static int32_t monBuildReqSql(char *sql) { - SStatisInfo info = dnodeGetStatisInfo(); - return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum); -} - -static int32_t monBuildIoSql(char *sql) { - float readKB = 0, writeKB = 0; - bool suc = taosGetProcIO(&readKB, &writeKB); - if (!suc) { - monDebug("failed to get io info"); - } - - return sprintf(sql, ", %f, %f", readKB, writeKB); -} - -static void monSaveSystemInfo() { - int64_t ts = taosGetTimestampUs(); - char * sql = tsMonitor.sql; - int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn%d values(%" PRId64, tsMonitorDbName, dnodeGetDnodeId(), ts); - - pos += monBuildCpuSql(sql + pos); - pos += monBuildMemorySql(sql + pos); - pos += monBuildDiskSql(sql + pos); - pos += monBuildBandSql(sql + pos); - pos += monBuildIoSql(sql + pos); - pos += monBuildReqSql(sql + pos); - - void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - int32_t code = taos_errno(res); - taos_free_result(res); - - if (code != 0) { - monError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); - } else { - monDebug("successfully to save system info, sql:%s", tsMonitor.sql); - } -} - -static void monExecSqlCb(void *param, TAOS_RES *result, int32_t code) { - int32_t c = taos_errno(result); - if (c != TSDB_CODE_SUCCESS) { - monError("save %s failed, reason:%s", (char *)param, tstrerror(c)); - } else { - int32_t rows = taos_affected_rows(result); - monDebug("save %s succ, rows:%d", (char *)param, rows); - } - - taos_free_result(result); -} - -void monSaveAcctLog(SAcctMonitorObj *pMon) { - if (tsMonitor.state != MON_STATE_INITED) return; - - char sql[1024] = {0}; - sprintf(sql, - "insert into %s.acct_%s using %s.acct tags('%s') values(now" - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %d)", - tsMonitorDbName, pMon->acctId, tsMonitorDbName, pMon->acctId, - pMon->currentPointsPerSecond, pMon->maxPointsPerSecond, - pMon->totalTimeSeries, pMon->maxTimeSeries, - pMon->totalStorage, pMon->maxStorage, - pMon->totalQueryTime, pMon->maxQueryTime, - pMon->totalInbound, pMon->maxInbound, - pMon->totalOutbound, pMon->maxOutbound, - pMon->totalDbs, pMon->maxDbs, - pMon->totalUsers, pMon->maxUsers, - pMon->totalStreams, pMon->maxStreams, - pMon->totalConns, pMon->maxConns, - pMon->accessState); - - monDebug("save account info, sql:%s", sql); - taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "account info"); -} - -void monSaveLog(int32_t level, const char *const format, ...) { - if (tsMonitor.state != MON_STATE_INITED) return; - - va_list argpointer; - char sql[SQL_LENGTH] = {0}; - int32_t max_length = SQL_LENGTH - 30; - int32_t len = snprintf(sql, (size_t)max_length, "insert into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName, - taosGetTimestampUs(), level); - - va_start(argpointer, format); - len += vsnprintf(sql + len, (size_t)(max_length - len), format, argpointer); - va_end(argpointer); - if (len > max_length) len = max_length; - - len += sprintf(sql + len, "', '%s')", tsLocalEp); - sql[len++] = 0; - - monDebug("save log, sql: %s", sql); - taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "log"); -} - -void monExecuteSQL(char *sql) { - if (tsMonitor.state != MON_STATE_INITED) return; - - monDebug("execute sql:%s", sql); - taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql"); -} - -void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param) { - if (tsMonitor.conn == NULL) { - callback(param, NULL, TSDB_CODE_MON_CONNECTION_INVALID); - return; - } - - monDebug("execute sql:%s", sql); - taos_query_a(tsMonitor.conn, sql, callback, param); -} diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 4627377a8e7b1dd1e732e033082d20bd4de622e2..85f961b82967c4ce2623f6dcaf8be928566394e1 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5142,22 +5142,22 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - pInfo->current = 0; + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + pInfo->current = 0; // pInfo->prevGroupId = -1; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableScanOperator"; - pOperator->operatorType = OP_TableScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doTableScan; + pOperator->name = "TableScanOperator"; + pOperator->operatorType = OP_TableScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScan; return pOperator; } @@ -5174,14 +5174,14 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE pRuntimeEnv->enableGroupData = true; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = OP_TableSeqScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doTableScanImpl; + pOperator->name = "TableSeqScanOperator"; + pOperator->operatorType = OP_TableSeqScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScanImpl; return pOperator; } @@ -5199,14 +5199,14 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRu taosArrayPush(pInfo->block.pDataBlock, &infoData); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableBlockInfoScanOperator"; - pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->exec = doBlockInfoScan; + pOperator->name = "TableBlockInfoScanOperator"; + pOperator->operatorType = OP_TableBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->exec = doBlockInfoScan; return pOperator; } @@ -5271,11 +5271,11 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->current = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->current = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); pOptr->name = "DataBlocksOptimizedScanOperator"; @@ -5429,14 +5429,14 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MERGE_STAGE); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "GlobalAggregate"; - pOperator->operatorType = OP_GlobalAggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->name = "GlobalAggregate"; + pOperator->operatorType = OP_GlobalAggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; @@ -5473,16 +5473,16 @@ SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SEx } SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiwaySortOperator"; - pOperator->operatorType = OP_MultiwayMergeSort; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = numOfOutput; - pOperator->pExpr = pExpr; - pOperator->exec = doMultiwayMergeSort; - pOperator->cleanup = destroyGlobalAggOperatorInfo; + pOperator->name = "MultiwaySortOperator"; + pOperator->operatorType = OP_MultiwayMergeSort; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = pExpr; + pOperator->exec = doMultiwayMergeSort; + pOperator->cleanup = destroyGlobalAggOperatorInfo; return pOperator; } @@ -6543,14 +6543,14 @@ SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpera setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MASTER_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableAggregate"; - pOperator->operatorType = OP_Aggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->name = "TableAggregate"; + pOperator->operatorType = OP_Aggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doAggregate; pOperator->cleanup = destroyAggOperatorInfo; @@ -6638,14 +6638,14 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SO initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiTableAggregate"; - pOperator->operatorType = OP_MultiTableAggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->name = "MultiTableAggregate"; + pOperator->operatorType = OP_MultiTableAggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSTableAggregate; pOperator->cleanup = destroyAggOperatorInfo; @@ -6668,14 +6668,14 @@ SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "ProjectOperator"; - pOperator->operatorType = OP_Project; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->name = "ProjectOperator"; + pOperator->operatorType = OP_Project; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doProjectOperation; pOperator->cleanup = destroyProjectOperatorInfo; @@ -6920,16 +6920,16 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "GroupbyAggOperator"; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Groupby; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashGroupbyAggregate; - pOperator->cleanup = destroyGroupbyOperatorInfo; + pOperator->name = "GroupbyAggOperator"; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Groupby; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = hashGroupbyAggregate; + pOperator->cleanup = destroyGroupbyOperatorInfo; appendUpstream(pOperator, upstream); return pOperator; @@ -7163,16 +7163,16 @@ SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInf pInfo->curPos = 0; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "SeqTableTagScan"; - pOperator->operatorType = OP_TagScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->exec = doTagScan; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyTagScanOperatorInfo; + pOperator->name = "SeqTableTagScan"; + pOperator->operatorType = OP_TagScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->exec = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroyTagScanOperatorInfo; return pOperator; } @@ -7302,17 +7302,17 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "DistinctOperator"; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Distinct; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashDistinct; - pOperator->pExpr = pExpr; - pOperator->cleanup = destroyDistinctOperatorInfo; + pOperator->name = "DistinctOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Distinct; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = hashDistinct; + pOperator->pExpr = pExpr; + pOperator->cleanup = destroyDistinctOperatorInfo; appendUpstream(pOperator, upstream); return pOperator; @@ -7587,20 +7587,20 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { param->pExpr[i] = pExprMsg; - pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->colBytes = htons(pExprMsg->colBytes); - pExprMsg->colType = htons(pExprMsg->colType); + pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->colBytes = htons(pExprMsg->colBytes); + pExprMsg->colType = htons(pExprMsg->colType); - pExprMsg->resType = htons(pExprMsg->resType); - pExprMsg->resBytes = htons(pExprMsg->resBytes); - pExprMsg->interBytes = htonl(pExprMsg->interBytes); + pExprMsg->resType = htons(pExprMsg->resType); + pExprMsg->resBytes = htons(pExprMsg->resBytes); + pExprMsg->interBytes = htonl(pExprMsg->interBytes); - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - pExprMsg->resColId = htons(pExprMsg->resColId); - pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); + pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + pExprMsg->resColId = htons(pExprMsg->resColId); + pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); pMsg += sizeof(SSqlExpr); for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { @@ -7639,15 +7639,15 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { param->pSecExpr[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->resType = htons(pExprMsg->resType); - pExprMsg->resBytes = htons(pExprMsg->resBytes); - pExprMsg->colBytes = htons(pExprMsg->colBytes); - pExprMsg->colType = htons(pExprMsg->colType); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->resType = htons(pExprMsg->resType); + pExprMsg->resBytes = htons(pExprMsg->resBytes); + pExprMsg->colBytes = htons(pExprMsg->colBytes); + pExprMsg->colType = htons(pExprMsg->colType); - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); pMsg += sizeof(SSqlExpr); @@ -8422,40 +8422,40 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S SQueryAttr* pQueryAttr = &pQInfo->query; pQInfo->runtimeEnv.pQueryAttr = pQueryAttr; - pQueryAttr->tableGroupInfo = *pTableGroupInfo; - pQueryAttr->numOfCols = numOfCols; - pQueryAttr->numOfOutput = numOfOutput; - pQueryAttr->limit.limit = pQueryMsg->limit; - pQueryAttr->limit.offset = pQueryMsg->offset; - pQueryAttr->order.order = pQueryMsg->order; - pQueryAttr->order.orderColId = pQueryMsg->orderColId; - pQueryAttr->pExpr1 = pExprs; - pQueryAttr->pExpr2 = pSecExprs; - pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput; - pQueryAttr->pGroupbyExpr = pGroupbyExpr; + pQueryAttr->tableGroupInfo = *pTableGroupInfo; + pQueryAttr->numOfCols = numOfCols; + pQueryAttr->numOfOutput = numOfOutput; + pQueryAttr->limit.limit = pQueryMsg->limit; + pQueryAttr->limit.offset = pQueryMsg->offset; + pQueryAttr->order.order = pQueryMsg->order; + pQueryAttr->order.orderColId = pQueryMsg->orderColId; + pQueryAttr->pExpr1 = pExprs; + pQueryAttr->pExpr2 = pSecExprs; + pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput; + pQueryAttr->pGroupbyExpr = pGroupbyExpr; memcpy(&pQueryAttr->interval, &pQueryMsg->interval, sizeof(pQueryAttr->interval)); - pQueryAttr->fillType = pQueryMsg->fillType; - pQueryAttr->numOfTags = pQueryMsg->numOfTags; - pQueryAttr->tagColList = pTagCols; + pQueryAttr->fillType = pQueryMsg->fillType; + pQueryAttr->numOfTags = pQueryMsg->numOfTags; + pQueryAttr->tagColList = pTagCols; pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; - pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; - pQueryAttr->sw = pQueryMsg->sw; - pQueryAttr->vgId = vgId; - - pQueryAttr->stableQuery = pQueryMsg->stableQuery; - pQueryAttr->topBotQuery = pQueryMsg->topBotQuery; - pQueryAttr->groupbyColumn = pQueryMsg->groupbyColumn; - pQueryAttr->hasTagResults = pQueryMsg->hasTagResults; + pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; + pQueryAttr->sw = pQueryMsg->sw; + pQueryAttr->vgId = vgId; + + pQueryAttr->stableQuery = pQueryMsg->stableQuery; + pQueryAttr->topBotQuery = pQueryMsg->topBotQuery; + pQueryAttr->groupbyColumn = pQueryMsg->groupbyColumn; + pQueryAttr->hasTagResults = pQueryMsg->hasTagResults; pQueryAttr->timeWindowInterpo = pQueryMsg->timeWindowInterpo; - pQueryAttr->queryBlockDist = pQueryMsg->queryBlockDist; - pQueryAttr->stabledev = pQueryMsg->stabledev; - pQueryAttr->tsCompQuery = pQueryMsg->tsCompQuery; - pQueryAttr->simpleAgg = pQueryMsg->simpleAgg; - pQueryAttr->pointInterpQuery = pQueryMsg->pointInterpQuery; - pQueryAttr->needReverseScan = pQueryMsg->needReverseScan; - pQueryAttr->stateWindow = pQueryMsg->stateWindow; - pQueryAttr->vgId = vgId; - pQueryAttr->pFilters = pFilters; + pQueryAttr->queryBlockDist = pQueryMsg->queryBlockDist; + pQueryAttr->stabledev = pQueryMsg->stabledev; + pQueryAttr->tsCompQuery = pQueryMsg->tsCompQuery; + pQueryAttr->simpleAgg = pQueryMsg->simpleAgg; + pQueryAttr->pointInterpQuery = pQueryMsg->pointInterpQuery; + pQueryAttr->needReverseScan = pQueryMsg->needReverseScan; + pQueryAttr->stateWindow = pQueryMsg->stateWindow; + pQueryAttr->vgId = vgId; + pQueryAttr->pFilters = pFilters; pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQueryAttr->tableCols == NULL) { diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h deleted file mode 100644 index 67e9976c704e15ae33174ff50ffdf1a289b5182a..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbMemTable.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_MEMTABLE_H_ -#define _TD_TSDB_MEMTABLE_H_ - -typedef struct { - int rowsInserted; - int rowsUpdated; - int rowsDeleteSucceed; - int rowsDeleteFailed; - int nOperations; - TSKEY keyFirst; - TSKEY keyLast; -} SMergeInfo; - -typedef struct { - STable * pTable; - SSkipListIterator *pIter; -} SCommitIter; - -struct STableData { - uint64_t uid; - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - SSkipList* pData; - T_REF_DECLARE() -}; - -enum { TSDB_UPDATE_META, TSDB_DROP_META }; - -#ifdef WINDOWS -#pragma pack(push ,1) -typedef struct { -#else -typedef struct __attribute__((packed)){ -#endif - char act; - uint64_t uid; -} SActObj; -#ifdef WINDOWS -#pragma pack(pop) -#endif - -typedef struct { - int len; - char cont[]; -} SActCont; - -int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot, SArray* pATable); -void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot); -void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); -int tsdbAsyncCommit(STsdbRepo* pRepo); -int tsdbSyncCommitConfig(STsdbRepo* pRepo); -int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); -void* tsdbCommitData(STsdbRepo* pRepo); - -static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator* pIter) { - if (pIter == NULL) return NULL; - - SSkipListNode* node = tSkipListIterGet(pIter); - if (node == NULL) return NULL; - - return (SMemRow)SL_GET_NODE_DATA(node); -} - -static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { - SMemRow row = tsdbNextIterRow(pIter); - if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; - - return memRowKey(row); -} - -static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { - SMemRow row = tsdbNextIterRow(pIter); - if (row == NULL) return TKEY_NULL; - - return memRowTKey(row); -} - -#endif /* _TD_TSDB_MEMTABLE_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c deleted file mode 100644 index 70589031f6516a129a5a683b0e76edb23b814e15..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbBuffer.c +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdbint.h" -#include "tsdbHealth.h" - -#define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0) - -// ---------------- INTERNAL FUNCTIONS ---------------- -STsdbBufPool *tsdbNewBufPool() { - STsdbBufPool *pBufPool = (STsdbBufPool *)calloc(1, sizeof(*pBufPool)); - if (pBufPool == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - int code = pthread_cond_init(&(pBufPool->poolNotEmpty), NULL); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } - - pBufPool->bufBlockList = tdListNew(sizeof(STsdbBufBlock *)); - if (pBufPool->bufBlockList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - return pBufPool; - -_err: - tsdbFreeBufPool(pBufPool); - return NULL; -} - -void tsdbFreeBufPool(STsdbBufPool *pBufPool) { - if (pBufPool) { - if (pBufPool->bufBlockList) { - ASSERT(listNEles(pBufPool->bufBlockList) == 0); - tdListFree(pBufPool->bufBlockList); - } - - pthread_cond_destroy(&pBufPool->poolNotEmpty); - - free(pBufPool); - } -} - -int tsdbOpenBufPool(STsdbRepo *pRepo) { - STsdbCfg * pCfg = &(pRepo->config); - STsdbBufPool *pPool = pRepo->pPool; - - ASSERT(pPool != NULL); - pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; // MB - pPool->tBufBlocks = pCfg->totalBlocks; - pPool->nBufBlocks = 0; - pPool->nElasticBlocks = 0; - pPool->index = 0; - pPool->nRecycleBlocks = 0; - - for (int i = 0; i < pCfg->totalBlocks; i++) { - STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); - if (pBufBlock == NULL) goto _err; - - if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { - tsdbFreeBufBlock(pBufBlock); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pPool->nBufBlocks++; - } - - tsdbDebug("vgId:%d buffer pool is opened! bufBlockSize:%d tBufBlocks:%d nBufBlocks:%d", REPO_ID(pRepo), - pPool->bufBlockSize, pPool->tBufBlocks, pPool->nBufBlocks); - - return 0; - -_err: - tsdbCloseBufPool(pRepo); - return -1; -} - -void tsdbCloseBufPool(STsdbRepo *pRepo) { - if (pRepo == NULL) return; - - STsdbBufPool * pBufPool = pRepo->pPool; - STsdbBufBlock *pBufBlock = NULL; - - if (pBufPool) { - SListNode *pNode = NULL; - while ((pNode = tdListPopHead(pBufPool->bufBlockList)) != NULL) { - tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); - tsdbFreeBufBlock(pBufBlock); - free(pNode); - } - } - - tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo)); -} - -SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { - ASSERT(pRepo != NULL && pRepo->pPool != NULL); - ASSERT(IS_REPO_LOCKED(pRepo)); - - STsdbBufPool *pBufPool = pRepo->pPool; - - while (POOL_IS_EMPTY(pBufPool)) { - if(tsDeadLockKillQuery) { - // supply new Block - if(tsdbInsertNewBlock(pRepo) > 0) { - tsdbWarn("vgId:%d add new elastic block . elasticBlocks=%d cur free Blocks=%d", REPO_ID(pRepo), pBufPool->nElasticBlocks, pBufPool->bufBlockList->numOfEles); - break; - } else { - // no newBlock, kill query free - if(!tsdbUrgeQueryFree(pRepo)) - tsdbWarn("vgId:%d Urge query free thread start failed.", REPO_ID(pRepo)); - } - } - - pRepo->repoLocked = false; - pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex)); - pRepo->repoLocked = true; - } - - SListNode * pNode = tdListPopHead(pBufPool->bufBlockList); - ASSERT(pNode != NULL); - STsdbBufBlock *pBufBlock = NULL; - tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); - - pBufBlock->blockId = pBufPool->index++; - pBufBlock->offset = 0; - pBufBlock->remain = pBufPool->bufBlockSize; - - tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId); - return pNode; -} - -// ---------------- LOCAL FUNCTIONS ---------------- -STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { - STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize); - if (pBufBlock == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pBufBlock->blockId = 0; - pBufBlock->offset = 0; - pBufBlock->remain = bufBlockSize; - - return pBufBlock; -} - - void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } - -int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { - if (oldTotalBlocks == pRepo->config.totalBlocks) { - return TSDB_CODE_SUCCESS; - } - - int err = TSDB_CODE_SUCCESS; - - if (tsdbLockRepo(pRepo) < 0) return terrno; - STsdbBufPool* pPool = pRepo->pPool; - - if (pRepo->config.totalBlocks > oldTotalBlocks) { - for (int i = 0; i < pRepo->config.totalBlocks - oldTotalBlocks; i++) { - STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); - if (pBufBlock == NULL) goto err; - - if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { - tsdbFreeBufBlock(pBufBlock); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto err; - } - - pPool->nBufBlocks++; - } - pthread_cond_signal(&pPool->poolNotEmpty); - } else { - pPool->nRecycleBlocks = oldTotalBlocks - pRepo->config.totalBlocks; - } - -err: - tsdbUnlockRepo(pRepo); - return err; -} - -void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic) { - STsdbBufBlock *pBufBlock = NULL; - tdListNodeGetData(pPool->bufBlockList, pNode, (void *)(&pBufBlock)); - tsdbFreeBufBlock(pBufBlock); - free(pNode); - if(bELastic) - { - pPool->nElasticBlocks--; - tsdbWarn("pPool=%p elastic block reduce one . nElasticBlocks=%d cur free Blocks=%d", pPool, pPool->nElasticBlocks, pPool->bufBlockList->numOfEles); - } - else - pPool->nBufBlocks--; -} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c deleted file mode 100644 index 3890dca5b96c26009dcf3ca72205ca4b1725aa29..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbMemTable.c +++ /dev/null @@ -1,1067 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdataformat.h" -#include "tfunctional.h" -#include "tsdbint.h" -#include "tskiplist.h" -#include "tsdbRowMergeBuf.h" - -#define TSDB_DATA_SKIPLIST_LEVEL 5 -#define TSDB_MAX_INSERT_BATCH 512 - -typedef struct { - int32_t totalLen; - int32_t len; - SMemRow row; -} SSubmitBlkIter; - -typedef struct { - int32_t totalLen; - int32_t len; - void * pMsg; -} SSubmitMsgIter; - -static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); -static void tsdbFreeMemTable(SMemTable *pMemTable); -static STableData* tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); -static void tsdbFreeTableData(STableData *pTableData); -static char * tsdbGetTsTupleKey(const void *data); -static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row); -static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); -static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); -static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg); -static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows); -static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); -static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock); -static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row); - -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, - TSKEY now); - -int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { - STsdbRepo * pRepo = repo; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk * pBlock = NULL; - int32_t affectedrows = 0; - - if (tsdbScanAndConvertSubmitMsg(pRepo, pMsg) < 0) { - if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { - tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno)); - } - return -1; - } - - tsdbInitSubmitMsgIter(pMsg, &msgIter); - while (true) { - tsdbGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - if (tsdbInsertDataToTable(pRepo, pBlock, &affectedrows) < 0) { - return -1; - } - } - - if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows); - - if (tsdbCheckCommit(pRepo) < 0) return -1; - return 0; -} - -// ---------------- INTERNAL FUNCTIONS ---------------- -int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - if (pMemTable == NULL) return 0; - int ref = T_REF_INC(pMemTable); - tsdbDebug("vgId:%d ref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); - return 0; -} - -// Need to lock the repository -int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - if (pMemTable == NULL) return 0; - - int ref = T_REF_DEC(pMemTable); - tsdbDebug("vgId:%d unref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); - if (ref == 0) { - STsdbBufPool *pBufPool = pRepo->pPool; - - SListNode *pNode = NULL; - bool addNew = false; - if (tsdbLockRepo(pRepo) < 0) return -1; - while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { - if (pBufPool->nRecycleBlocks > 0) { - tsdbRecycleBufferBlock(pBufPool, pNode, false); - pBufPool->nRecycleBlocks -= 1; - } else { - if(pBufPool->nElasticBlocks > 0 && listNEles(pBufPool->bufBlockList) > 2) { - tsdbRecycleBufferBlock(pBufPool, pNode, true); - } else { - tdListAppendNode(pBufPool->bufBlockList, pNode); - addNew = true; - } - } - } - if (addNew) { - int code = pthread_cond_signal(&pBufPool->poolNotEmpty); - if (code != 0) { - if (tsdbUnlockRepo(pRepo) < 0) return -1; - tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - } - - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - for (int i = 0; i < pMemTable->maxTables; i++) { - if (pMemTable->tData[i] != NULL) { - tsdbFreeTableData(pMemTable->tData[i]); - } - } - - tdListDiscard(pMemTable->actList); - tdListDiscard(pMemTable->bufBlockList); - tsdbFreeMemTable(pMemTable); - } - return 0; -} - -int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot, SArray *pATable) { - memset(pSnapshot, 0, sizeof(*pSnapshot)); - - if (tsdbLockRepo(pRepo) < 0) return -1; - - pSnapshot->omem = pRepo->mem; - pSnapshot->imem = pRepo->imem; - tsdbRefMemTable(pRepo, pRepo->mem); - tsdbRefMemTable(pRepo, pRepo->imem); - - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - if (pSnapshot->omem) { - taosRLockLatch(&(pSnapshot->omem->latch)); - - pSnapshot->mem = &(pSnapshot->mtable); - - pSnapshot->mem->tData = (STableData **)calloc(pSnapshot->omem->maxTables, sizeof(STableData *)); - if (pSnapshot->mem->tData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - taosRUnLockLatch(&(pSnapshot->omem->latch)); - tsdbUnRefMemTable(pRepo, pSnapshot->omem); - tsdbUnRefMemTable(pRepo, pSnapshot->imem); - pSnapshot->mem = NULL; - pSnapshot->imem = NULL; - pSnapshot->omem = NULL; - return -1; - } - - pSnapshot->mem->keyFirst = pSnapshot->omem->keyFirst; - pSnapshot->mem->keyLast = pSnapshot->omem->keyLast; - pSnapshot->mem->numOfRows = pSnapshot->omem->numOfRows; - pSnapshot->mem->maxTables = pSnapshot->omem->maxTables; - - for (size_t i = 0; i < taosArrayGetSize(pATable); i++) { - STable * pTable = *(STable **)taosArrayGet(pATable, i); - int32_t tid = TABLE_TID(pTable); - STableData *pTableData = (tid < pSnapshot->omem->maxTables) ? pSnapshot->omem->tData[tid] : NULL; - - if ((pTableData == NULL) || (TABLE_UID(pTable) != pTableData->uid)) continue; - - pSnapshot->mem->tData[tid] = pTableData; - T_REF_INC(pTableData); - } - - taosRUnLockLatch(&(pSnapshot->omem->latch)); - } - - tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem); - return 0; -} - -void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot) { - tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem); - - if (pSnapshot->mem) { - ASSERT(pSnapshot->omem != NULL); - - for (size_t i = 0; i < pSnapshot->mem->maxTables; i++) { - STableData *pTableData = pSnapshot->mem->tData[i]; - if (pTableData) { - tsdbFreeTableData(pTableData); - } - } - tfree(pSnapshot->mem->tData); - - tsdbUnRefMemTable(pRepo, pSnapshot->omem); - } - - tsdbUnRefMemTable(pRepo, pSnapshot->imem); - - pSnapshot->mem = NULL; - pSnapshot->imem = NULL; - pSnapshot->omem = NULL; -} - -void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { - STsdbCfg * pCfg = &pRepo->config; - STsdbBufBlock *pBufBlock = NULL; - void * ptr = NULL; - - // Either allocate from buffer blocks or from SYSTEM memory pool - if (pRepo->mem == NULL) { - SMemTable *pMemTable = tsdbNewMemTable(pRepo); - if (pMemTable == NULL) return NULL; - pRepo->mem = pMemTable; - } - - ASSERT(pRepo->mem != NULL); - - pBufBlock = tsdbGetCurrBufBlock(pRepo); - if ((pRepo->mem->extraBuffList != NULL) || - ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < bytes))) { - // allocate from SYSTEM buffer pool - if (pRepo->mem->extraBuffList == NULL) { - pRepo->mem->extraBuffList = tdListNew(0); - if (pRepo->mem->extraBuffList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - } - - ASSERT(pRepo->mem->extraBuffList != NULL); - SListNode *pNode = (SListNode *)malloc(sizeof(SListNode) + bytes); - if (pNode == NULL) { - if (listNEles(pRepo->mem->extraBuffList) == 0) { - tdListFree(pRepo->mem->extraBuffList); - pRepo->mem->extraBuffList = NULL; - } - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pNode->next = pNode->prev = NULL; - tdListAppendNode(pRepo->mem->extraBuffList, pNode); - ptr = (void *)(pNode->data); - tsdbTrace("vgId:%d allocate %d bytes from SYSTEM buffer block", REPO_ID(pRepo), bytes); - } else { // allocate from TSDB buffer pool - if (pBufBlock == NULL || pBufBlock->remain < bytes) { - ASSERT(listNEles(pRepo->mem->bufBlockList) < pCfg->totalBlocks / 3); - if (tsdbLockRepo(pRepo) < 0) return NULL; - SListNode *pNode = tsdbAllocBufBlockFromPool(pRepo); - tdListAppendNode(pRepo->mem->bufBlockList, pNode); - if (tsdbUnlockRepo(pRepo) < 0) return NULL; - pBufBlock = tsdbGetCurrBufBlock(pRepo); - } - - ASSERT(pBufBlock->remain >= bytes); - ptr = POINTER_SHIFT(pBufBlock->data, pBufBlock->offset); - pBufBlock->offset += bytes; - pBufBlock->remain -= bytes; - tsdbTrace("vgId:%d allocate %d bytes from TSDB buffer block, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes, - listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); - } - - return ptr; -} - -int tsdbSyncCommitConfig(STsdbRepo* pRepo) { - ASSERT(pRepo->config_changed == true); - tsem_wait(&(pRepo->readyToCommit)); - - if (pRepo->code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d try to commit config when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); - } - - if (tsdbLockRepo(pRepo) < 0) return -1; - tsdbScheduleCommit(pRepo, COMMIT_CONFIG_REQ); - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - tsem_wait(&(pRepo->readyToCommit)); - tsem_post(&(pRepo->readyToCommit)); - - if (pRepo->code != TSDB_CODE_SUCCESS) { - terrno = pRepo->code; - return -1; - } - - terrno = TSDB_CODE_SUCCESS; - return 0; -} - -int tsdbAsyncCommit(STsdbRepo *pRepo) { - tsem_wait(&(pRepo->readyToCommit)); - - ASSERT(pRepo->imem == NULL); - if (pRepo->mem == NULL) { - tsem_post(&(pRepo->readyToCommit)); - return 0; - } - - if (pRepo->code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); - } - - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START, TSDB_CODE_SUCCESS); - if (tsdbLockRepo(pRepo) < 0) return -1; - pRepo->imem = pRepo->mem; - pRepo->mem = NULL; - tsdbScheduleCommit(pRepo, COMMIT_REQ); - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - return 0; -} - -int tsdbSyncCommit(STsdbRepo *repo) { - STsdbRepo *pRepo = repo; - - tsdbAsyncCommit(pRepo); - tsem_wait(&(pRepo->readyToCommit)); - tsem_post(&(pRepo->readyToCommit)); - - if (pRepo->code != TSDB_CODE_SUCCESS) { - terrno = pRepo->code; - return -1; - } else { - terrno = TSDB_CODE_SUCCESS; - return 0; - } -} - -/** - * This is an important function to load data or try to load data from memory skiplist iterator. - * - * This function load memory data until: - * 1. iterator ends - * 2. data key exceeds maxKey - * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead - * 4. operations in pCols not exceeds its max capacity if pCols is given - * - * The function tries to procceed AS MUCH AS POSSIBLE. - */ -int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { - ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); - if (pIter == NULL) return 0; - STSchema * pSchema = NULL; - TSKEY rowKey = 0; - TSKEY fKey = 0; - bool isRowDel = false; - int filterIter = 0; - SMemRow row = NULL; - SMergeInfo mInfo; - - if (pMergeInfo == NULL) pMergeInfo = &mInfo; - - memset(pMergeInfo, 0, sizeof(*pMergeInfo)); - pMergeInfo->keyFirst = INT64_MAX; - pMergeInfo->keyLast = INT64_MIN; - if (pCols) tdResetDataCols(pCols); - - row = tsdbNextIterRow(pIter); - if (row == NULL || memRowKey(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = memRowKey(row); - isRowDel = memRowDeleted(row); - } - - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - - while (true) { - if (fKey == INT64_MAX && rowKey == INT64_MAX) break; - - if (fKey < rowKey) { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); - - filterIter++; - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - } else if (fKey > rowKey) { - if (isRowDel) { - pMergeInfo->rowsDeleteFailed++; - } else { - if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsInserted++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || memRowKey(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = memRowKey(row); - isRowDel = memRowDeleted(row); - } - } else { - if (isRowDel) { - ASSERT(!keepDup); - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsDeleteSucceed++; - pMergeInfo->nOperations++; - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - if (keepDup) { - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsUpdated++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); - } - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || memRowKey(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = memRowKey(row); - isRowDel = memRowDeleted(row); - } - - filterIter++; - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - } - } - - return 0; -} - -// ---------------- LOCAL FUNCTIONS ---------------- -static SMemTable* tsdbNewMemTable(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - SMemTable *pMemTable = (SMemTable *)calloc(1, sizeof(*pMemTable)); - if (pMemTable == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMemTable->keyFirst = INT64_MAX; - pMemTable->keyLast = 0; - pMemTable->numOfRows = 0; - - pMemTable->maxTables = pMeta->maxTables; - pMemTable->tData = (STableData **)calloc(pMemTable->maxTables, sizeof(STableData *)); - if (pMemTable->tData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMemTable->actList = tdListNew(0); - if (pMemTable->actList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMemTable->bufBlockList = tdListNew(sizeof(STsdbBufBlock*)); - if (pMemTable->bufBlockList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - T_REF_INC(pMemTable); - - return pMemTable; - -_err: - tsdbFreeMemTable(pMemTable); - return NULL; -} - -static void tsdbFreeMemTable(SMemTable* pMemTable) { - if (pMemTable) { - ASSERT((pMemTable->bufBlockList == NULL) ? true : (listNEles(pMemTable->bufBlockList) == 0)); - ASSERT((pMemTable->actList == NULL) ? true : (listNEles(pMemTable->actList) == 0)); - - tdListFree(pMemTable->extraBuffList); - tdListFree(pMemTable->bufBlockList); - tdListFree(pMemTable->actList); - tfree(pMemTable->tData); - free(pMemTable); - } -} - -static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { - STableData *pTableData = (STableData *)calloc(1, sizeof(*pTableData)); - if (pTableData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pTableData->uid = TABLE_UID(pTable); - pTableData->keyFirst = INT64_MAX; - pTableData->keyLast = 0; - pTableData->numOfRows = 0; - - uint8_t skipListCreateFlags; - if(pCfg->update == TD_ROW_DISCARD_UPDATE) - skipListCreateFlags = SL_DISCARD_DUP_KEY; - else - skipListCreateFlags = SL_UPDATE_DUP_KEY; - - pTableData->pData = - tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], - tkeyComparFn, skipListCreateFlags, tsdbGetTsTupleKey); - if (pTableData->pData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - free(pTableData); - return NULL; - } - - T_REF_INC(pTableData); - - return pTableData; -} - -static void tsdbFreeTableData(STableData *pTableData) { - if (pTableData) { - int32_t ref = T_REF_DEC(pTableData); - if (ref == 0) { - tSkipListDestroy(pTableData->pData); - free(pTableData); - } - } -} - -static char *tsdbGetTsTupleKey(const void *data) { return memRowTuple((SMemRow)data); } - -static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { - ASSERT(pMemTable->maxTables < maxTables); - - STableData **pTableData = (STableData **)calloc(maxTables, sizeof(STableData *)); - if (pTableData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - memcpy((void *)pTableData, (void *)pMemTable->tData, sizeof(STableData *) * pMemTable->maxTables); - - STableData **tData = pMemTable->tData; - - taosWLockLatch(&(pMemTable->latch)); - pMemTable->maxTables = maxTables; - pMemTable->tData = pTableData; - taosWUnLockLatch(&(pMemTable->latch)); - - tfree(tData); - - return 0; -} - -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row) { - if (pCols) { - if (*ppSchema == NULL || schemaVersion(*ppSchema) != memRowVersion(row)) { - *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row)); - if (*ppSchema == NULL) { - ASSERT(false); - return -1; - } - } - - tdAppendMemRowToDataCol(row, *ppSchema, pCols, true); - } - - return 0; -} - -static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { - if (pBlock->dataLen <= 0) return -1; - pIter->totalLen = pBlock->dataLen; - pIter->len = 0; - pIter->row = (SMemRow)(pBlock->data + pBlock->schemaLen); - return 0; -} - -static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { - SMemRow row = pIter->row; // firstly, get current row - if (row == NULL) return NULL; - - pIter->len += memRowTLen(row); - if (pIter->len >= pIter->totalLen) { // reach the end - pIter->row = NULL; - } else { - pIter->row = (char *)row + memRowTLen(row); // secondly, move to next row - } - - return row; -} - -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, - TSKEY now) { - TSKEY rowKey = memRowKey(row); - if (rowKey < minKey || rowKey > maxKey) { - tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 - " maxKey %" PRId64 " row key %" PRId64, - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, - rowKey); - terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - return -1; - } - - return 0; -} - -static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { - ASSERT(pMsg != NULL); - STsdbMeta * pMeta = pRepo->tsdbMeta; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk * pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - SMemRow row = NULL; - TSKEY now = taosGetTimestamp(pRepo->config.precision); - TSKEY minKey = now - tsTickPerDay[pRepo->config.precision] * pRepo->config.keep; - TSKEY maxKey = now + tsTickPerDay[pRepo->config.precision] * pRepo->config.daysPerFile; - - terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tsdbGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; - - pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); - - if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { - tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - STable *pTable = pMeta->tables[pBlock->tid]; - if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) { - tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable)); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - // Check schema version and update schema if needed - if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) { - if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - continue; - } else { - return -1; - } - } - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pRepo, pTable, row, minKey, maxKey, now) < 0) { - return -1; - } - } - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} - -//row1 has higher priority -static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, - STSchema **ppSchema1, STSchema **ppSchema2, - STable* pTable, int32_t* pPoints, SMemRow* pLastRow) { - - //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! - if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) { - (*pPoints)++; - return NULL; - } - - if(row2 == NULL || pRepo->config.update != TD_ROW_PARTIAL_UPDATE) { - void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1)); - if(pMem == NULL) return NULL; - memRowCpy(pMem, row1); - (*pPoints)++; - *pLastRow = pMem; - return pMem; - } - - STSchema *pSchema1 = *ppSchema1; - STSchema *pSchema2 = *ppSchema2; - SMergeBuf * pBuf = &pRepo->mergeBuf; - int dv1 = memRowVersion(row1); - int dv2 = memRowVersion(row2); - if(pSchema1 == NULL || schemaVersion(pSchema1) != dv1) { - if(pSchema2 != NULL && schemaVersion(pSchema2) == dv1) { - *ppSchema1 = pSchema2; - } else { - *ppSchema1 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row1)); - } - pSchema1 = *ppSchema1; - } - - if(pSchema2 == NULL || schemaVersion(pSchema2) != dv2) { - if(schemaVersion(pSchema1) == dv2) { - pSchema2 = pSchema1; - } else { - *ppSchema2 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row2)); - pSchema2 = *ppSchema2; - } - } - - SMemRow tmp = tsdbMergeTwoRows(pBuf, row1, row2, pSchema1, pSchema2); - - void* pMem = tsdbAllocBytes(pRepo, memRowTLen(tmp)); - if(pMem == NULL) return NULL; - memRowCpy(pMem, tmp); - - (*pPoints)++; - *pLastRow = pMem; - return pMem; -} - -static void* tsdbInsertDupKeyMergePacked(void** args) { - return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]); -} - -static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, SMemRow* pLastRow) { - - if(pSkipList->insertHandleFn == NULL) { - tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9); - dupHandleSavedFunc->args[2] = pRepo; - dupHandleSavedFunc->args[3] = NULL; - dupHandleSavedFunc->args[4] = NULL; - dupHandleSavedFunc->args[5] = pTable; - pSkipList->insertHandleFn = dupHandleSavedFunc; - } - pSkipList->insertHandleFn->args[6] = pPoints; - pSkipList->insertHandleFn->args[7] = pLastRow; -} - -static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *pAffectedRows) { - - STsdbMeta *pMeta = pRepo->tsdbMeta; - int32_t points = 0; - STable *pTable = NULL; - SSubmitBlkIter blkIter = {0}; - SMemTable *pMemTable = NULL; - STableData *pTableData = NULL; - STsdbCfg *pCfg = &(pRepo->config); - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - if(blkIter.row == NULL) return 0; - TSKEY firstRowKey = memRowKey(blkIter.row); - - tsdbAllocBytes(pRepo, 0); - pMemTable = pRepo->mem; - - ASSERT(pMemTable != NULL); - ASSERT(pBlock->tid < pMeta->maxTables); - - pTable = pMeta->tables[pBlock->tid]; - - ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); - - - if (TABLE_TID(pTable) >= pMemTable->maxTables) { - if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) { - return -1; - } - } - pTableData = pMemTable->tData[TABLE_TID(pTable)]; - - if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) { - if (pTableData != NULL) { - taosWLockLatch(&(pMemTable->latch)); - pMemTable->tData[TABLE_TID(pTable)] = NULL; - tsdbFreeTableData(pTableData); - taosWUnLockLatch(&(pMemTable->latch)); - } - - pTableData = tsdbNewTableData(pCfg, pTable); - if (pTableData == NULL) { - tsdbError("vgId:%d failed to insert data to table %s uid %" PRId64 " tid %d since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), tstrerror(terrno)); - return -1; - } - - pRepo->mem->tData[TABLE_TID(pTable)] = pTableData; - } - - ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); - - SMemRow lastRow = NULL; - int64_t osize = SL_SIZE(pTableData->pData); - tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow); - tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext); - int64_t dsize = SL_SIZE(pTableData->pData) - osize; - (*pAffectedRows) += points; - - - if(lastRow != NULL) { - TSKEY lastRowKey = memRowKey(lastRow); - if (pMemTable->keyFirst > firstRowKey) pMemTable->keyFirst = firstRowKey; - pMemTable->numOfRows += dsize; - - if (pTableData->keyFirst > firstRowKey) pTableData->keyFirst = firstRowKey; - pTableData->numOfRows += dsize; - if (pMemTable->keyLast < lastRowKey) pMemTable->keyLast = lastRowKey; - if (pTableData->keyLast < lastRowKey) pTableData->keyLast = lastRowKey; - if (tsdbUpdateTableLatestInfo(pRepo, pTable, lastRow) < 0) { - return -1; - } - } - - STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion); - pRepo->stat.pointsWritten += points * schemaNCols(pSchema); - pRepo->stat.totalStorage += points * schemaVLen(pSchema); - - return 0; -} - - -static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { - if (pMsg == NULL) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - - pIter->totalLen = pMsg->length; - pIter->len = 0; - pIter->pMsg = pMsg; - if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - - return 0; -} - -static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { - if (pIter->len == 0) { - pIter->len += TSDB_SUBMIT_MSG_HEAD_SIZE; - } else { - SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen); - } - - if (pIter->len > pIter->totalLen) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - *pPBlock = NULL; - return -1; - } - - *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - - return 0; -} - -static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) { - ASSERT(pTable != NULL); - - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - int sversion = schemaVersion(pSchema); - - if (pBlock->sversion == sversion) { - return 0; - } else { - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { // stream table is not allowed to change schema - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; - } - } - - if (pBlock->sversion > sversion) { // may need to update table schema - if (pBlock->schemaLen > 0) { - tsdbDebug( - "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion); - ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0); - int numOfCols = pBlock->schemaLen / sizeof(STColumn); - STColumn *pTCol = (STColumn *)pBlock->data; - - STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - return -1; - } - - for (int i = 0; i < numOfCols; i++) { - if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - tdDestroyTSchemaBuilder(&schemaBuilder); - return -1; - } - } - - STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNSchema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tdDestroyTSchemaBuilder(&schemaBuilder); - return -1; - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true); - } else { - tsdbDebug( - "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion); - terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE; - return -1; - } - } else { - ASSERT(pBlock->sversion >= 0); - if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) { - tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo), - pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable)); - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; - } - } - - return 0; -} - - -static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow row) { - tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, - memRowVersion(row)); - - STSchema* pSchema = tsdbGetTableLatestSchema(pTable); - if (tsdbUpdateLastColSchema(pTable, pSchema) < 0) { - return; - } - - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); - if (pSchema == NULL) { - return; - } - - SDataCol *pLatestCols = pTable->lastCols; - int32_t kvIdx = 0; - - for (int16_t j = 0; j < schemaNCols(pSchema); j++) { - STColumn *pTCol = schemaColAt(pSchema, j); - // ignore not exist colId - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pTCol->colId); - if (idx == -1) { - continue; - } - - void *value = NULL; - - value = tdGetMemRowDataOfColEx(row, pTCol->colId, (int8_t)pTCol->type, - TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset, &kvIdx); - - if ((value == NULL) || isNull(value, pTCol->type)) { - continue; - } - - SDataCol *pDataCol = &(pLatestCols[idx]); - if (pDataCol->pData == NULL) { - pDataCol->pData = malloc(pTCol->bytes); - pDataCol->bytes = pTCol->bytes; - } else if (pDataCol->bytes < pTCol->bytes) { - pDataCol->pData = realloc(pDataCol->pData, pTCol->bytes); - pDataCol->bytes = pTCol->bytes; - } - // the actual value size - uint16_t bytes = IS_VAR_DATA_TYPE(pTCol->type) ? varDataTLen(value) : pTCol->bytes; - // the actual data size CANNOT larger than column size - assert(pTCol->bytes >= bytes); - memcpy(pDataCol->pData, value, bytes); - //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData); - pDataCol->ts = memRowKey(row); - } -} - -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row) { - STsdbCfg *pCfg = &pRepo->config; - - // if cacheLastRow config has been reset, free the lastRow - if (!pCfg->cacheLastRow && pTable->lastRow != NULL) { - SMemRow cachedLastRow = pTable->lastRow; - TSDB_WLOCK_TABLE(pTable); - pTable->lastRow = NULL; - TSDB_WUNLOCK_TABLE(pTable); - taosTZfree(cachedLastRow); - } - - if (tsdbGetTableLastKeyImpl(pTable) <= memRowKey(row)) { - if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) { - SMemRow nrow = pTable->lastRow; - if (taosTSizeof(nrow) < memRowTLen(row)) { - SMemRow orow = nrow; - nrow = taosTMalloc(memRowTLen(row)); - if (nrow == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - memRowCpy(nrow, row); - TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = memRowKey(row); - pTable->lastRow = nrow; - TSDB_WUNLOCK_TABLE(pTable); - taosTZfree(orow); - } else { - TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = memRowKey(row); - memRowCpy(nrow, row); - TSDB_WUNLOCK_TABLE(pTable); - } - } else { - pTable->lastKey = memRowKey(row); - } - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - updateTableLatestColumn(pRepo, pTable, row); - } - } - return 0; -} diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt deleted file mode 100644 index 0d9be42bd5d54ddd1fdd372511e4f98fb7d6355b..0000000000000000000000000000000000000000 --- a/src/wal/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -ADD_DEFINITIONS(-DWAL_CHECKSUM_WHOLE) - -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) - -ADD_LIBRARY(twal ${SRC}) -TARGET_LINK_LIBRARIES(twal tutil common) -ADD_SUBDIRECTORY(test) diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt deleted file mode 100644 index c5bc4198f10d48caf2ea133c475ea99c8e7a2fd2..0000000000000000000000000000000000000000 --- a/src/wal/test/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -IF (TD_LINUX) - INCLUDE_DIRECTORIES(../inc) - - LIST(APPEND WALTEST_SRC ./waltest.c) - ADD_EXECUTABLE(waltest ${WALTEST_SRC}) - TARGET_LINK_LIBRARIES(waltest twal os tutil) - -ENDIF () - -IF (TD_DARWIN) - INCLUDE_DIRECTORIES(../inc) - - LIST(APPEND WALTEST_SRC ./waltest.c) - ADD_EXECUTABLE(waltest ${WALTEST_SRC}) - TARGET_LINK_LIBRARIES(waltest twal os tutil) - -ENDIF () -