diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a05cc0224a1a4c9d82579721cd7870ec81766686..326f00dbbd5c4565c03b3c8b253d511d42b7668c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,6 @@ ADD_SUBDIRECTORY(client) ADD_SUBDIRECTORY(query) ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(plugins) -ADD_SUBDIRECTORY(sdb) ADD_SUBDIRECTORY(mnode) ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(dnode) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 298eb6110c8dab7ac34a24446d851b37a6b5dace..d85782b463f4ca579c9874ee1050593ff01f40ca 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode sdb taos_static monitor http tsdb) + TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb) #IF (TD_CLUSTER) # TARGET_LINK_LIBRARIES(taosd dcluster) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index be152fbf3cbc6000cb40affbe99ea07e3c62b05e..792a8ec1cce02ed0b00ef5d44b32624ccfb9ca0e 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "taosmsg.h" #include "taoserror.h" -#include "sdb.h" #include "tglobalcfg.h" #include "thash.h" #include "tidpool.h" @@ -54,8 +53,8 @@ typedef struct { int8_t numOfMnodes; int32_t numOfDnodes; char mnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; int syncFd; void *hbTimer; void *pSync; @@ -79,8 +78,8 @@ typedef struct { float lbScore; // calc in balance function int32_t customScore; // config by user char dnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; SVnodeLoad vload[TSDB_MAX_VNODES]; int32_t status; uint32_t lastReboot; // time stamp for last reboot @@ -121,7 +120,7 @@ typedef struct SSuperTableObj { int32_t sversion; int32_t numOfColumns; int32_t numOfTags; - int8_t reserved[5]; + int8_t reserved[15]; int8_t updateEnd[1]; int32_t numOfTables; int16_t nextColId; @@ -153,26 +152,29 @@ typedef struct { int32_t sversion; int32_t numOfColumns; int32_t sqlLen; - int8_t reserved[3]; + int8_t reserved[7]; int8_t updateEnd[1]; char* sql; //null-terminated string int16_t nextColId; SSchema* schema; } SNormalTableObj; +struct _db_obj; + typedef struct _vg_obj { uint32_t vgId; char dbName[TSDB_DB_NAME_LEN + 1]; int64_t createdTime; SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT]; int32_t numOfVnodes; - int32_t numOfTables; int32_t lbIp; int32_t lbTime; int8_t lbStatus; int8_t reserved[14]; int8_t updateEnd[1]; struct _vg_obj *prev, *next; + struct _db_obj *pDb; + int32_t numOfTables; void * idPool; STableInfo ** tableList; } SVgObj; @@ -182,8 +184,8 @@ typedef struct _db_obj { int8_t dirty; int64_t createdTime; SDbCfg cfg; - char reserved[15]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; struct _db_obj *prev, *next; int32_t numOfVgroups; int32_t numOfTables; @@ -201,7 +203,7 @@ typedef struct _user_obj { int64_t createdTime; int8_t superAuth; int8_t writeAuth; - int8_t reserved[16]; + int8_t reserved[13]; int8_t updateEnd[1]; struct _user_obj *prev, *next; struct _acctObj * pAcct; diff --git a/src/inc/sdb.h b/src/inc/sdb.h deleted file mode 100644 index 4b4de1ac4b9b2575493f289f98660dbbb6704911..0000000000000000000000000000000000000000 --- a/src/inc/sdb.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SDB_H -#define TDENGINE_SDB_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosmsg.h" -#include "taosdef.h" - -extern uint16_t tsMgmtMgmtPort; -extern uint16_t tsMgmtSyncPort; -extern int tsMgmtPeerHBTimer; // seconds -extern char * sdbStatusStr[]; -extern char * sdbRoleStr[]; -extern int sdbMaster; -extern SRpcIpSet *pSdbIpList; -extern SRpcIpSet *pSdbPublicIpList; - -extern void (*sdbWorkAsMasterCallback)(); // this function pointer will be set by taosd - -enum _keytype { - SDB_KEYTYPE_STRING, SDB_KEYTYPE_UINT32, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_RECYCLE, SDB_KEYTYPE_MAX -}; - -#define SDB_ROLE_UNAPPROVED 0 -#define SDB_ROLE_UNDECIDED 1 -#define SDB_ROLE_MASTER 2 -#define SDB_ROLE_SLAVE 3 - -#define SDB_STATUS_OFFLINE 0 -#define SDB_STATUS_UNSYNCED 1 -#define SDB_STATUS_SYNCING 2 -#define SDB_STATUS_SERVING 3 -#define SDB_STATUS_DELETED 4 - -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, - SDB_TYPE_DECODE, - SDB_TYPE_ENCODE, - SDB_TYPE_BEFORE_BATCH_UPDATE, - SDB_TYPE_BATCH_UPDATE, - SDB_TYPE_AFTER_BATCH_UPDATE, - SDB_TYPE_RESET, - SDB_TYPE_DESTROY, - SDB_MAX_ACTION_TYPES -}; - -#define SDB_MAX_PEERS 4 -typedef struct { - uint32_t ip; - uint32_t publicIp; - char ipstr[20]; - char zone[12]; - char role; - int64_t createdTime; - uint64_t dbVersion; - int64_t lostTime; - char status; - char numOfMnodes; - int numOfDnodes; - char updateEnd[1]; - - // internal - int syncFd; - void *hbTimer; - void *pSync; -} SSdbPeer; - -extern SSdbPeer *sdbPeer[]; -#define sdbInited (sdbPeer[0]) -#define sdbStatus (sdbPeer[0]->status) - -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)); - -void *sdbGetRow(void *handle, void *key); - -int64_t sdbInsertRow(void *handle, void *row, int rowSize); - -int sdbDeleteRow(void *handle, void *key); - -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated); - -void *sdbFetchRow(void *handle, void *pNode, void **ppRow); - -int sdbBatchUpdateRow(void *handle, void *row, int rowSize); - -int64_t sdbGetId(void *handle); - -int64_t sdbGetNumOfRows(void *handle); - -void sdbSaveSnapShot(void *handle); - -void sdbCloseTable(void *handle); - -int sdbRemovePeerByIp(uint32_t ip); - -int sdbInitPeers(char *directory); - -void sdbCleanUpPeers(); - -int64_t sdbGetVersion(); - -int32_t sdbGetRunStatus(); - -#define TSDB_MAX_NORMAL_TABLES 10000 -#define TSDB_MAX_SUPER_TABLES 1000 - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SDB_H diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 5da8aa191b5e1ce5072a80599a0c64b3f44d9f15..570e2e1acd0391f2f59f56b9b68b52e38310da14 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -309,6 +309,16 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_SESSIONS_PER_VNODE (300) #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES) +#define TSDB_MAX_MNODES 5 +#define TSDB_MAX_DNODES 10 +#define TSDB_MAX_ACCOUNTS 10 +#define TSDB_MAX_USERS 20 +#define TSDB_MAX_DBS 100 +#define TSDB_MAX_VGROUPS 1000 +#define TSDB_MAX_SUPER_TABLES 100 +#define TSDB_MAX_NORMAL_TABLES 1000 +#define TSDB_MAX_CHILD_TABLES 100000 + enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 66e8cf73988ab25db7544b9a52215d2279630c63..386c8a92f781670f38580cbc794e4e0d3044fce4 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) -ADD_SUBDIRECTORY(taosdemo) -ADD_SUBDIRECTORY(taosdump) +#ADD_SUBDIRECTORY(taosdemo) +#ADD_SUBDIRECTORY(taosdump) diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 5bf4cfd604c81f14a95129bcd6887d2ca2904b4f..bacd7497ad5832b963db7d188550e4ce560829c7 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) - TARGET_LINK_LIBRARIES(mnode trpc tutil sdb pthread) + TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) IF (TD_CLUSTER) TARGET_LINK_LIBRARIES(mnode) diff --git a/src/mnode/inc/mgmtChildTable.h b/src/mnode/inc/mgmtChildTable.h index b16dd58f67402e84c2c611282fded15abea88e97..9252a7d485ae02055de81835f2c8c0a0d0da5cbe 100644 --- a/src/mnode/inc/mgmtChildTable.h +++ b/src/mnode/inc/mgmtChildTable.h @@ -39,6 +39,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); void mgmtDropAllChildTables(SDbObj *pDropDb); +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index 32bb9d9ec63f386200f85f00d2c5d92c07b04273..f2049f836ceb20a775b25927c91ef040cda78338 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -22,11 +22,11 @@ extern "C" { #include "mnode.h" -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup); +void mgmtAddVgroupIntoDb(SVgObj *pVgroup); +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup); +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup); +void mgmtMoveVgroupToTail(SVgObj *pVgroup); +void mgmtMoveVgroupToHead(SVgObj *pVgroup); int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index d768d2dd7cd7ca517a02d5dcce88545b178e2851..729d31544f4c49f729e8c6d958a5a038661e225b 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -20,9 +20,15 @@ extern "C" { #endif -bool mgmtCheckRedirect(void *handle); +int32_t mgmtInitMnodes(); +void mgmtCleanupMnodes(); + +bool mgmtInServerStatus(); +bool mgmtIsMaster(); -void mgmtGetMnodeIpList(SRpcIpSet *ipSet); +bool mgmtCheckRedirect(void *handle); +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp); int32_t mgmtRemoveMnode(uint32_t privateIp); diff --git a/src/mnode/inc/mgmtProfile.h b/src/mnode/inc/mgmtProfile.h index 7f9fd9622c7b782e1d796fc36f6340198702934b..8c2e073c2d5981d54f6ca8fc41144bb03f8467b2 100644 --- a/src/mnode/inc/mgmtProfile.h +++ b/src/mnode/inc/mgmtProfile.h @@ -28,6 +28,8 @@ bool mgmtCheckQhandle(uint64_t qhandle); void mgmtSaveQhandle(void *qhandle); void mgmtFreeQhandle(void *qhandle); +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h new file mode 100644 index 0000000000000000000000000000000000000000..ccddc11e04fc32cf2ee8cd0b503c51a993025d4d --- /dev/null +++ b/src/mnode/inc/mgmtSdb.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_MNODE_SDB_H +#define TDENGINE_MNODE_SDB_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + SDB_KEY_TYPE_STRING, + SDB_KEY_TYPE_AUTO +} ESdbKeyType; + +typedef enum { + SDB_OPER_TYPE_GLOBAL, + SDB_OPER_TYPE_LOCAL +} ESdbOperType; + +typedef struct { + ESdbOperType type; + void * table; + void * pObj; + int64_t version; + int32_t maxRowSize; + int32_t rowSize; + void * rowData; +} SSdbOperDesc; + +typedef struct { + char *tableName; + int32_t hashSessions; + int32_t maxRowSize; + ESdbKeyType keyType; + int32_t (*insertFp)(SSdbOperDesc *pOper); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pDesc); + int32_t (*destroyFp)(SSdbOperDesc *pDesc); +} SSdbTableDesc; + +void *sdbOpenTable(SSdbTableDesc *desc); +void sdbCloseTable(void *handle); + +int32_t sdbInsertRow(SSdbOperDesc *pOper); +int32_t sdbDeleteRow(SSdbOperDesc *pOper); +int32_t sdbUpdateRow(SSdbOperDesc *pOper); + +void *sdbGetRow(void *handle, void *key); +void *sdbFetchRow(void *handle, void *pNode, void **ppRow); +int64_t sdbGetNumOfRows(void *handle); +uint64_t sdbGetVersion(); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index b7c68b5f8062240fd1f7c71f1e789d3a31f08630..3379a93f28f0dc8933fcff3cf51bc587c01cc444 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -28,13 +28,12 @@ int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode); +void mgmtDropAllVgroups(SDbObj *pDropDb); void mgmtCreateVgroup(SQueuedMsg *pMsg); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); void mgmtUpdateVgroup(SVgObj *pVgroup); -void mgmtUpdateVgroupIp(SDnodeObj *pDnode); - -void mgmtSetVgroupIdPool(); +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb); void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable); diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index b1ee72386ef9bc36635c982685e7885bd1ab5f93..0075f53319cf63f92457c02204d4d1ab098706c5 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -85,8 +85,10 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { pUser->prev->next = pUser->next; } - if (pUser->next) pUser->next->prev = pUser->prev; - + if (pUser->next) { + pUser->next->prev = pUser->prev; + } + if (pUser->prev == NULL) { pAcct->pUser = pUser->next; } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index d5f0e7c85378d2b775a323a3d1a1bad783088a3f..9076681caa3dfc5e28f85f4c36ca96ff01978165 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -25,79 +25,47 @@ #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" +#include "mgmtDClient.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" -#include "mgmtDClient.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" void *tsChildTableSdb; int32_t tsChildTableUpdateSize; -void *(*mgmtChildTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static void mgmtDestroyChildTable(SChildTableObj *pTable) { - free(pTable); -} - -static void mgmtChildTableActionInit() { - mgmtChildTableActionFp[SDB_TYPE_INSERT] = mgmtChildTableActionInsert; - mgmtChildTableActionFp[SDB_TYPE_DELETE] = mgmtChildTableActionDelete; - mgmtChildTableActionFp[SDB_TYPE_UPDATE] = mgmtChildTableActionUpdate; - mgmtChildTableActionFp[SDB_TYPE_ENCODE] = mgmtChildTableActionEncode; - mgmtChildTableActionFp[SDB_TYPE_DECODE] = mgmtChildTableActionDecode; - mgmtChildTableActionFp[SDB_TYPE_RESET] = mgmtChildTableActionReset; - mgmtChildTableActionFp[SDB_TYPE_DESTROY] = mgmtChildTableActionDestroy; -} - -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - memcpy(pTable, str, tsChildTableUpdateSize); - return NULL; + tfree(pTable); } -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *)row; - mgmtDestroyChildTable(pTable); - return NULL; +static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroyChildTable(pOper->pObj); + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("ctable:%s, not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in db:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; - } - - if (!sdbMaster) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); - return NULL; - } + return TSDB_CODE_INVALID_ACCT; } pTable->superTable = mgmtGetSuperTable(pTable->superTableId); @@ -107,83 +75,65 @@ void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ss mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); - } - - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; + return TSDB_CODE_INVALID_ACCT; } mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - mgmtRemoveTableFromSuperTable(pTable->superTable); - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); - } - - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtChildTableActionReset(row, str, size, NULL); +static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; + assert(pTable != NULL && pOper->rowData != NULL); - memcpy(str, pTable, tsChildTableUpdateSize); - *ssize = tsChildTableUpdateSize; + memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); + pOper->rowSize = tsChildTableUpdateSize; - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); - - SChildTableObj *pTable = (SChildTableObj *)calloc(sizeof(SChildTableObj), 1); - if (pTable == NULL) return NULL; +static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - if (size < tsChildTableUpdateSize) { - mgmtDestroyChildTable(pTable); - return NULL; + pOper->pObj = calloc(1, sizeof(SChildTableObj)); + if (pOper->pObj == NULL) { + return TSDB_CODE_SERV_OUT_OF_MEMORY; } - memcpy(pTable, str, tsChildTableUpdateSize); - return (void *)pTable; -} + memcpy(pOper->pObj, pOper->rowData, tsChildTableUpdateSize); -void *mgmtChildTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtChildTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtChildTableActionFp[(uint8_t)action]))(row, str, size, ssize); - } - return NULL; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitChildTables() { @@ -191,12 +141,23 @@ int32_t mgmtInitChildTables() { void *pLastNode = NULL; SChildTableObj *pTable = NULL; - mgmtChildTableActionInit(); SChildTableObj tObj; - tsChildTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "ctables", + .hashSessions = tsMaxTables, + .maxRowSize = tsChildTableUpdateSize, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtChildTableActionInsert, + .deleteFp = mgmtChildTableActionDelete, + .updateFp = mgmtChildTableActionUpdate, + .encodeFp = mgmtChildTableActionEncode, + .decodeFp = mgmtChildTableActionDecode, + .destroyFp = mgmtChildTableActionDestroy, + }; - tsChildTableSdb = sdbOpenTable(tsMaxTables, tsChildTableUpdateSize, - "ctables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtChildTableAction); + tsChildTableSdb = sdbOpenTable(&tableDesc); if (tsChildTableSdb == NULL) { mError("failed to init child table data"); return -1; @@ -212,7 +173,11 @@ int32_t mgmtInitChildTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -221,7 +186,11 @@ int32_t mgmtInitChildTables() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -230,7 +199,11 @@ int32_t mgmtInitChildTables() { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -238,28 +211,27 @@ int32_t mgmtInitChildTables() { if (pVgroup->tableList == NULL) { mError("ctable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } - pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); - SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); if (pSuperTable == NULL) { mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } - - pTable->superTable = pSuperTable; - mgmtAddTableIntoSuperTable(pSuperTable); - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); } mTrace("child table is initialized"); @@ -271,8 +243,13 @@ void mgmtCleanUpChildTables() { } void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { - char *pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; - int32_t tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + char * pTagData = NULL; + int32_t tagDataLen = 0; + if (pMsg != NULL) { + pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; + tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + } + int32_t totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen; @@ -305,18 +282,14 @@ void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTab pSchema++; } - memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + if (pMsg != NULL) { + memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + } + return pCreate; } void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { - int32_t numOfTables = sdbGetNumOfRows(tsChildTableSdb); - if (numOfTables >= tsMaxTables) { - mError("ctable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, tsMaxTables); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - char *pTagData = (char *) pCreate->schema; // it is a tag key SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); if (pSuperTable == NULL) { @@ -337,12 +310,18 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->type = TSDB_CHILD_TABLE; pTable->createdTime = taosGetTimestampMs(); pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + - ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = tid; pTable->vgId = pVgroup->vgId; pTable->superTable = pSuperTable; - if (sdbInsertRow(tsChildTableSdb, pTable, 0) < 0) { + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbInsertRow(&desc); + + if (sdbInsertRow(&desc) < 0) { free(pTable); mError("ctable:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_SDB_ERROR; @@ -485,12 +464,45 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; + } + } + + mTrace("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); +} + +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + SChildTableObj *pTable = NULL; + + while (1) { + pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); + if (pTable == NULL) { + break; + } + + if (pTable->superTable == pStable) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; continue; } } - mTrace("db:%s, all child tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->tableId, numOfTables); } \ No newline at end of file diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index da11ad20811734d158aeaa12733ddae178402273..cbdcb8ff5dec6fae0684f8257a5379058622618e 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -42,7 +42,7 @@ int32_t mgmtInitDClient() { rpcInit.label = "MND-DC"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessRspFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "mgmtDClient"; diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 177a45764ccd773a3ceb549fd0dccb624c4257c3..b1d01b91f1f147b600fc70bd696be8e986c3eadf 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -45,7 +45,7 @@ int32_t mgmtInitDServer() { rpcInit.label = "MND-DS"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessMsgFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = mgmtDServerRetrieveAuth; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 88934599dc237b6451d8631b36db5583c37e819e..0cc793f2c077218ca97cf3dad5230c456022dc52 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -15,33 +15,33 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "mgmtDb.h" +#include "taoserror.h" +#include "tstatus.h" +#include "tutil.h" +#include "name.h" +#include "mnode.h" #include "mgmtAcct.h" #include "mgmtBalance.h" #include "mgmtChildTable.h" +#include "mgmtDb.h" #include "mgmtDnode.h" #include "mgmtGrant.h" +#include "mgmtShell.h" #include "mgmtMnode.h" #include "mgmtNormalTable.h" -#include "mgmtShell.h" +#include "mgmtChildTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" -#include "mnode.h" - -#include "taoserror.h" -#include "tstatus.h" -#include "tutil.h" -#include "name.h" static void *tsDbSdb = NULL; static int32_t tsDbUpdateSize; static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(void *handle, void *tmrId); -static void mgmtSetDbDirty(SDbObj *pDb); +static int32_t mgmtSetDbDirty(SDbObj *pDb); static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -49,65 +49,93 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg); static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg); -static void *(*mgmtDbActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - -static void mgmtDbActionInit() { - mgmtDbActionFp[SDB_TYPE_INSERT] = mgmtDbActionInsert; - mgmtDbActionFp[SDB_TYPE_DELETE] = mgmtDbActionDelete; - mgmtDbActionFp[SDB_TYPE_UPDATE] = mgmtDbActionUpdate; - mgmtDbActionFp[SDB_TYPE_ENCODE] = mgmtDbActionEncode; - mgmtDbActionFp[SDB_TYPE_DECODE] = mgmtDbActionDecode; - mgmtDbActionFp[SDB_TYPE_RESET] = mgmtDbActionReset; - mgmtDbActionFp[SDB_TYPE_DESTROY] = mgmtDbActionDestroy; +static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtDbAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtDbActionFp[(uint8_t)action] != NULL) { - return (*(mgmtDbActionFp[(uint8_t)action]))(row, str, size, ssize); +static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + + pDb->pHead = NULL; + pDb->pTail = NULL; + pDb->prev = NULL; + pDb->next = NULL; + pDb->numOfVgroups = 0; + pDb->numOfTables = 0; + pDb->numOfSuperTables = 0; + + if (pAcct != NULL) { + mgmtAddDbIntoAcct(pAcct, pDb); + } + else { + mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; } - return NULL; + + return TSDB_CODE_SUCCESS; } -int32_t mgmtInitDbs() { - void * pNode = NULL; - SDbObj * pDb = NULL; - SAcctObj *pAcct = NULL; +static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtDbActionInit(); + mgmtRemoveDbFromAcct(pAcct, pDb); + mgmtDropAllNormalTables(pDb); + mgmtDropAllChildTables(pDb); + mgmtDropAllSuperTables(pDb); + mgmtDropAllVgroups(pDb); + + return TSDB_CODE_SUCCESS; +} - SDbObj tObj; - tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; +static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; +} - tsDbSdb = sdbOpenTable(tsMaxDbs, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); - if (tsDbSdb == NULL) { - mError("failed to init db data"); +static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + + if (pOper->maxRowSize < tsDbUpdateSize) { return -1; + } else { + memcpy(pOper->rowData, pDb, tsDbUpdateSize); + pOper->rowSize = tsDbUpdateSize; + return TSDB_CODE_SUCCESS; } +} - while (1) { - pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb); - if (pDb == NULL) break; +static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { + SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj)); + if (pDb == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; + + memcpy(pDb, pOper->rowData, tsDbUpdateSize); + pOper->pObj = pDb; + return TSDB_CODE_SUCCESS; +} - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->prev = NULL; - pDb->next = NULL; - pDb->numOfTables = 0; - pDb->numOfVgroups = 0; - pDb->numOfSuperTables = 0; - pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct != NULL) - mgmtAddDbIntoAcct(pAcct, pDb); - else { - mError("db:%s acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); - } +int32_t mgmtInitDbs() { + SDbObj tObj; + tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "dbs", + .hashSessions = TSDB_MAX_DBS, + .maxRowSize = tsDbUpdateSize, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtDbActionInsert, + .deleteFp = mgmtDbActionDelete, + .updateFp = mgmtDbActionUpdate, + .encodeFp = mgmtDbActionEncode, + .decodeFp = mgmtDbActionDecode, + .destroyFp = mgmtDbActionDestroy, + }; + + tsDbSdb = sdbOpenTable(&tableDesc); + if (tsDbSdb == NULL) { + mError("failed to init db data"); + return -1; } mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DB, mgmtProcessCreateDbMsg); @@ -210,14 +238,14 @@ static int32_t mgmtCheckDBParams(SCMCreateDbMsg *pCreate) { static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { // assign default parameters - if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; // - if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; // - if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; // - if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; // - if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; // - if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; // - if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; // - if (pCreate->compression < 0) pCreate->compression = tsCompression; // + if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; + if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; + if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; + if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; + if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; + if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; + if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; + if (pCreate->compression < 0) pCreate->compression = tsCompression; if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog; if (pCreate->replications < 0) pCreate->replications = tsReplications; // if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; // @@ -248,18 +276,10 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { pCreate->blocksPerTable = TSDB_MIN_AVG_BLOCKS; } - pCreate->maxSessions++; - return TSDB_CODE_SUCCESS; } static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { - int32_t numOfDbs = sdbGetNumOfRows(tsDbSdb); - if (numOfDbs >= tsMaxDbs) { - mWarn("numOfDbs:%d, exceed tsMaxDbs:%d", numOfDbs, tsMaxDbs); - return TSDB_CODE_TOO_MANY_DATABASES; - } - int32_t code = mgmtCheckDbLimit(pAcct); if (code != 0) { return code; @@ -287,9 +307,17 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->createdTime = taosGetTimestampMs(); pDb->cfg = *pCreate; - if (sdbInsertRow(tsDbSdb, pDb, 0) < 0) { - code = TSDB_CODE_SDB_ERROR; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pDb); + code = TSDB_CODE_SDB_ERROR; } return code; @@ -303,73 +331,8 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb)); } -static int32_t mgmtAlterDb(SAcctObj *pAcct, SCMAlterDbMsg *pAlter) { - return 0; -// int32_t code = TSDB_CODE_SUCCESS; -// -// SDbObj *pDb = (SDbObj *) sdbGetRow(tsDbSdb, pAlter->db); -// if (pDb == NULL) { -// mTrace("db:%s is not exist", pAlter->db); -// return TSDB_CODE_INVALID_DB; -// } -// -// int32_t oldReplicaNum = pDb->cfg.replications; -// if (pAlter->daysToKeep > 0) { -// mTrace("db:%s daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, pAlter->daysToKeep); -// pDb->cfg.daysToKeep = pAlter->daysToKeep; -// } else if (pAlter->replications > 0) { -// mTrace("db:%s replica:%d change to %d", pDb->name, pDb->cfg.replications, pAlter->replications); -// if (pAlter->replications < TSDB_REPLICA_MIN_NUM || pAlter->replications > TSDB_REPLICA_MAX_NUM) { -// mError("invalid db option replica: %d valid range: %d--%d", pAlter->replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); -// return TSDB_CODE_INVALID_OPTION; -// } -// pDb->cfg.replications = pAlter->replications; -// } else if (pAlter->maxSessions > 0) { -// mTrace("db:%s tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, pAlter->maxSessions); -// if (pAlter->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pAlter->maxSessions > TSDB_MAX_TABLES_PER_VNODE) { -// mError("invalid db option tables: %d valid range: %d--%d", pAlter->maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); -// return TSDB_CODE_INVALID_OPTION; -// } -// if (pAlter->maxSessions < pDb->cfg.maxSessions) { -// mError("invalid db option tables: %d should larger than original:%d", pAlter->maxSessions, pDb->cfg.maxSessions); -// return TSDB_CODE_INVALID_OPTION; -// } -// return TSDB_CODE_INVALID_OPTION; -// //The modification of tables needs to rewrite the head file, so disable this option -// //pDb->cfg.maxSessions = pAlter->maxSessions; -// } else { -// mError("db:%s alter msg, replica:%d, keep:%d, tables:%d, origin replica:%d keep:%d", pDb->name, -// pAlter->replications, pAlter->maxSessions, pAlter->daysToKeep, -// pDb->cfg.replications, pDb->cfg.daysToKeep); -// return TSDB_CODE_INVALID_OPTION; -// } -// -// if (sdbUpdateRow(tsDbSdb, pDb, tsDbUpdateSize, 1) < 0) { -// return TSDB_CODE_SDB_ERROR; -// } -// -// SVgObj *pVgroup = pDb->pHead; -// while (pVgroup != NULL) { -// balanceUpdateVgroupState(pVgroup, TSDB_VG_LB_STATUS_UPDATE, 0); -// if (oldReplicaNum < pDb->cfg.replications) { -// if (!balanceAddVnode(pVgroup, NULL, NULL)) { -// mWarn("db:%s vgroup:%d not enough dnode to add vnode", pAlter->db, pVgroup->vgId); -// code = TSDB_CODE_NO_ENOUGH_DNODES; -// } -// } -// if (pAlter->maxSessions > 0) { -// //rebuild meterList in mgmtVgroup.c -// mgmtUpdateVgroup(pVgroup); -// } -//// mgmtSendCreateVnodeMsg(pVgroup); -// pVgroup = pVgroup->next; -// } -// mgmtStartBalanceTimer(10); -// -// return code; -} - -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = pDb->pHead; pVgroup->prev = NULL; @@ -378,11 +341,10 @@ int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { pDb->pHead = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = NULL; pVgroup->prev = pDb->pTail; @@ -391,32 +353,25 @@ int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { pDb->pTail = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; if (pVgroup->prev) pVgroup->prev->next = pVgroup->next; if (pVgroup->next) pVgroup->next->prev = pVgroup->prev; if (pVgroup->prev == NULL) pDb->pHead = pVgroup->next; if (pVgroup->next == NULL) pDb->pTail = pVgroup->prev; pDb->numOfVgroups--; - - return 0; } -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToTail(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDbTail(pVgroup); } -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDb(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToHead(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDb(pVgroup); } void mgmtCleanUpDbs() { @@ -438,7 +393,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -566,7 +521,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -char *mgmtGetDbStr(char *src) { +static char *mgmtGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); return ++pos; @@ -636,7 +591,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * if (strcmp(pUser->user, "root") == 0) { #endif pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxSessions - 1; // table num can be created should minus 1 + *(int32_t *)pWrite = pDb->cfg.maxSessions; // table num can be created should minus 1 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -649,9 +604,9 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; #ifdef _TD_ARM_32_ - *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #else - *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #endif cols++; @@ -691,68 +646,6 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * return numOfRows; } -void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->numOfVgroups = 0; - pDb->numOfTables = 0; - mgmtAddDbIntoAcct(pAcct, pDb); - - return NULL; -} - -void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtRemoveDbFromAcct(pAcct, pDb); - - mgmtDropAllNormalTables(pDb); - mgmtDropAllChildTables(pDb); - mgmtDropAllSuperTables(pDb); - - return NULL; -} - -void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtDbActionReset(row, str, size, ssize); -} - -void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - if (size < tsDbUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pDb, tsDbUpdateSize); - *ssize = tsDbUpdateSize; - } - - return NULL; -} -void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); - if (pDb == NULL) return NULL; - memset(pDb, 0, sizeof(SDbObj)); - - memcpy(pDb, str, tsDbUpdateSize); - - return (void *)pDb; -} - -void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - memcpy(pDb, str, tsDbUpdateSize); - - return NULL; -} - -void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - return NULL; -} - void mgmtAddSuperTableIntoDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfSuperTables, 1); } @@ -768,8 +661,23 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfTables, -1); } -static void mgmtSetDbDirty(SDbObj *pDb) { +static int32_t mgmtSetDbDirty(SDbObj *pDb) { + if (pDb->dirty) return TSDB_CODE_SUCCESS; + pDb->dirty = true; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + + return code; } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { @@ -794,32 +702,121 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } else { code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate); if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is created by %s", pCreate->db, pMsg->pUser->user); + mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user); } } mgmtSendSimpleResp(pMsg->thandle, code); } +static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = pDb->cfg; + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t maxSessions = htonl(pAlter->maxSessions); + int8_t replications = pAlter->replications; + + terrno = TSDB_CODE_SUCCESS; + + if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { + mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); + newCfg.daysToKeep = daysToKeep; + } else if (replications > 0 && replications != pDb->cfg.replications) { + mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); + if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { + mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.replications = replications; + } else if (maxSessions > 0 && maxSessions != pDb->cfg.maxSessions) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, maxSessions); + if (maxSessions < TSDB_MIN_TABLES_PER_VNODE || maxSessions > TSDB_MAX_TABLES_PER_VNODE) { + mError("invalid db option tables: %d valid range: %d--%d", maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + terrno = TSDB_CODE_INVALID_OPTION; + } + if (maxSessions < pDb->cfg.maxSessions) { + mError("invalid db option tables: %d should larger than original:%d", maxSessions, pDb->cfg.maxSessions); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.maxSessions = maxSessions; + } else { + } + + return newCfg; +} + +static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = mgmtGetAlterDbOption(pDb, pAlter); + if (terrno != TSDB_CODE_SUCCESS) { + return terrno; + } + + if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { + pDb->cfg = newCfg; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + } + + return TSDB_CODE_SUCCESS; +} + static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) { if (mgmtCheckRedirect(pMsg->thandle)) return; SCMAlterDbMsg *pAlter = pMsg->pCont; - pAlter->daysPerFile = htonl(pAlter->daysPerFile); - pAlter->daysToKeep = htonl(pAlter->daysToKeep); - pAlter->maxSessions = htonl(pAlter->maxSessions) + 1; + mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle); + + if (mgmtCheckExpired()) { + mError("db:%s, failed to alter, grant expired", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); + return; + } - int32_t code; if (!pMsg->pUser->writeAuth) { - code = TSDB_CODE_NO_RIGHTS; - } else { - code = mgmtAlterDb(pMsg->pUser->pAcct, pAlter); - if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is altered by %s", pAlter->db, pMsg->pUser->user); - } + mError("db:%s, failed to alter, no rights", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); + return; } - mgmtSendSimpleResp(pMsg->thandle, code); + SDbObj *pDb = mgmtGetDb(pAlter->db); + if (pDb == NULL) { + mError("db:%s, failed to alter, invalid db", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + return; + } + + int32_t code = mgmtAlterDb(pDb, pAlter); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to alter, invalid db option", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, code); + } + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + SVgObj *pVgroup = pDb->pHead; + if (pVgroup != NULL) { + mPrint("vgroup:%d, will be altered", pVgroup->vgId); + newMsg->ahandle = pVgroup; + newMsg->expected = pVgroup->numOfVnodes; + mgmtAlterVgroup(pVgroup, newMsg); + return; + } + + mTrace("db:%s, all vgroups is altered", pDb->name); + + mgmtSendSimpleResp(newMsg->thandle, TSDB_CODE_SUCCESS); + rpcFreeCont(newMsg->pCont); + free(newMsg); } static void mgmtDropDb(void *handle, void *tmrId) { @@ -827,7 +824,12 @@ static void mgmtDropDb(void *handle, void *tmrId) { SDbObj *pDb = newMsg->ahandle; mPrint("db:%s, drop db from sdb", pDb->name); - int32_t code = sdbDeleteRow(tsDbSdb, pDb); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb + }; + int32_t code = sdbDeleteRow(&oper); if (code != 0) { code = TSDB_CODE_SDB_ERROR; } @@ -874,7 +876,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { return; } - mgmtSetDbDirty(pDb); + int32_t code = mgmtSetDbDirty(pDb); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code)); + mgmtSendSimpleResp(pMsg->thandle, code); + return; + } SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index cf57b43918fb5fd89665d6aee770007a0768a1d5..0aab0834718a2c4215ff6cdc10499e4f35edc4c9 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -547,9 +547,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return ; } - uint32_t lastPrivateIp = pDnode->privateIp; - uint32_t lastPublicIp = pDnode->publicIp; - pDnode->privateIp = htonl(pStatus->privateIp); pDnode->publicIp = htonl(pStatus->publicIp); pDnode->lastReboot = htonl(pStatus->lastReboot); @@ -566,11 +563,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mgmtSetDnodeMaxVnodes(pDnode); } - if (lastPrivateIp != pDnode->privateIp || lastPublicIp != pDnode->publicIp) { - mgmtUpdateVgroupIp(pDnode); - //mgmtUpdateMnodeIp(); - } - int32_t openVnodes = htons(pStatus->openVnodes); for (int32_t j = 0; j < openVnodes; ++j) { pDnode->vload[j].vgId = htonl(pStatus->load[j].vgId); @@ -599,7 +591,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mgmtGetMnodeIpList(&pRsp->ipList); + mgmtGetMnodePrivateIpList(&pRsp->ipList); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 66200e5a1407aa59e1e2a76f736c23eb0f82a7b3..ab0b99fa586512e64d1550b29ab77e1b4cbe4332 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -25,6 +25,8 @@ #include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtDServer.h" +#include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtVgroup.h" #include "mgmtUser.h" #include "mgmtTable.h" @@ -65,7 +67,7 @@ int32_t mgmtStartSystem() { return 0; } - tsMgmtTmr = taosTmrInit((tsMaxDnodes + tsMaxShellConns) * 3, 200, 3600000, "MND"); + tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND"); if (tsMgmtTmr == NULL) { mError("failed to init timer"); return -1; @@ -109,8 +111,8 @@ int32_t mgmtStartSystem() { return -1; } - if (sdbInitPeers(tsMnodeDir) < 0) { - mError("failed to init peers"); + if (mgmtInitMnodes() < 0) { + mError("failed to init mnodes"); return -1; } @@ -125,7 +127,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (sdbMaster) { + if (mgmtIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } @@ -136,7 +138,7 @@ void mgmtStopSystem() { void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); - sdbCleanUpPeers(); + mgmtCleanupMnodes(); mgmtCleanupBalance(); mgmtCleanUpShell(); mgmtCleanupDClient(); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index f7e6e3f8cb4c984917a30ff7000e7d4373619c3b..3dfce5e6afa452558bdd1c522361684b13a147f6 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -17,48 +17,93 @@ #include "os.h" #include "trpc.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtUser.h" -int32_t (*mgmtAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; -int32_t (*mgmtRemoveMnodeFp)(uint32_t privateIp) = NULL; -int32_t (*mgmtGetMnodesNumFp)() = NULL; -void * (*mgmtGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; +int32_t (*mpeerRemoveMnodeFp)(uint32_t privateIp) = NULL; +int32_t (*mpeerGetMnodesNumFp)() = NULL; +void * (*mpeerGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerInitMnodesFp)() = NULL; +void (*mpeerCleanUpMnodesFp)() = NULL; static SMnodeObj tsMnodeObj = {0}; +static bool tsMnodeIsMaster = false; +static bool tsMnodeIsServing = false; static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static char *mgmtMnodeStatusStr[] = { + "offline", + "unsynced", + "syncing", + "serving", + "null" +}; + +static char *mgmtMnodeRoleStr[] = { + "unauthed", + "undecided", + "master", + "slave", + "null" +}; + +int32_t mgmtInitMnodes() { + if (mpeerInitMnodesFp) { + return (*mpeerInitMnodesFp)(); + } else { + tsMnodeIsServing = true; + tsMnodeIsMaster = true; + return 0; + } +} + +void mgmtCleanupMnodes() { + if (mpeerCleanUpMnodesFp) { + (*mpeerCleanUpMnodesFp)(); + } +} + +bool mgmtInServerStatus() { + return tsMnodeIsServing; +} + +bool mgmtIsMaster() { + return tsMnodeIsMaster; +} + bool mgmtCheckRedirect(void *handle) { return false; } int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp) { - if (mgmtAddMnodeFp) { - return (*mgmtAddMnodeFp)(privateIp, publicIp); + if (mpeerAddMnodeFp) { + return (*mpeerAddMnodeFp)(privateIp, publicIp); } else { return 0; } } int32_t mgmtRemoveMnode(uint32_t privateIp) { - if (mgmtRemoveMnodeFp) { - return (*mgmtRemoveMnodeFp)(privateIp); + if (mpeerRemoveMnodeFp) { + return (*mpeerRemoveMnodeFp)(privateIp); } else { return 0; } } static int32_t mgmtGetMnodesNum() { - if (mgmtGetMnodesNumFp) { - return (*mgmtGetMnodesNumFp)(); + if (mpeerGetMnodesNumFp) { + return (*mpeerGetMnodesNumFp)(); } else { return 1; } } static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { - if (mgmtGetNextMnodeFp) { - return (*mgmtGetNextMnodeFp)(pShow, pMnode); + if (mpeerGetNextMnodeFp) { + return (*mpeerGetNextMnodeFp)(pShow, pMnode); } else { if (*pMnode == NULL) { *pMnode = &tsMnodeObj; @@ -148,11 +193,11 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbStatusStr[(uint8_t)pMnode->status]); + strcpy(pWrite, mgmtMnodeStatusStr[pMnode->status]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbRoleStr[(uint8_t)pMnode->role]); + strcpy(pWrite, mgmtMnodeRoleStr[pMnode->role]); cols++; tinet_ntoa(ipstr, pMnode->publicIp); @@ -167,7 +212,14 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -void mgmtGetMnodeIpList(SRpcIpSet *ipSet) { +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(inet_addr(tsMasterIp)); +} + +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index f076b69f36323f08c0e69195d6d1ab08b4ac4150..4e5e4ce7cc8af2bb99685632dd2b3388a3c247d4 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -26,191 +26,132 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtNormalTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" -void *tsNormalTableSdb; +void *tsNormalTableSdb; int32_t tsNormalTableUpdateSize; -void *(*mgmtNormalTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { - free(pTable->schema); - free(pTable->sql); - free(pTable); -} - -static void mgmtNormalTableActionInit() { - mgmtNormalTableActionFp[SDB_TYPE_INSERT] = mgmtNormalTableActionInsert; - mgmtNormalTableActionFp[SDB_TYPE_DELETE] = mgmtNormalTableActionDelete; - mgmtNormalTableActionFp[SDB_TYPE_UPDATE] = mgmtNormalTableActionUpdate; - mgmtNormalTableActionFp[SDB_TYPE_ENCODE] = mgmtNormalTableActionEncode; - mgmtNormalTableActionFp[SDB_TYPE_DECODE] = mgmtNormalTableActionDecode; - mgmtNormalTableActionFp[SDB_TYPE_RESET] = mgmtNormalTableActionReset; - mgmtNormalTableActionFp[SDB_TYPE_DESTROY] = mgmtNormalTableActionDestroy; -} - -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - memcpy(pTable, str, tsNormalTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; - pTable->schema = realloc(pTable->schema, schemaSize); - pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); - - return NULL; + tfree(pTable->schema); + tfree(pTable->sql); + tfree(pTable); } -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *)row; - mgmtDestroyNormalTable(pTable); - return NULL; +static int32_t mgmtNormalTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroyNormalTable(pOper->pObj); + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; +static int32_t mgmtNormalTableActionInsert(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("id:%s not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + mError("ntable:%s not in vgroup:%d", pTable->tableId, pTable->vgId); + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; + mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { - mError("account not exists"); - return NULL; - } - - if (!sdbMaster) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("sid:%d is not matched from the master:%d", sid, pTable->sid); - return NULL; - } + mError("ntable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; } mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); - } - - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; +static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; + mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("account not exists"); - return NULL; + return TSDB_CODE_INVALID_ACCT; } mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); - } - - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtNormalTableActionReset(row, str, size, NULL); +static int32_t mgmtNormalTableActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtNormalTableActionEncode(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (size < tsNormalTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(str, pTable, tsNormalTableUpdateSize); - memcpy(str + tsNormalTableUpdateSize, pTable->schema, schemaSize); - memcpy(str + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - *ssize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; + memcpy(pOper->rowData, pTable, tsNormalTableUpdateSize); + memcpy(pOper->rowData + tsNormalTableUpdateSize, pTable->schema, schemaSize); + memcpy(pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - return NULL; + pOper->rowSize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); - if (pTable == NULL) { - return NULL; - } - memset(pTable, 0, sizeof(SNormalTableObj)); + SNormalTableObj *pTable = (SNormalTableObj *)calloc(1, sizeof(SNormalTableObj)); + if (pTable == NULL) TSDB_CODE_SERV_OUT_OF_MEMORY; - if (size < tsNormalTableUpdateSize) { - mgmtDestroyNormalTable(pTable); - return NULL; - } - memcpy(pTable, str, tsNormalTableUpdateSize); + memcpy(pTable, pOper->rowData, tsNormalTableUpdateSize); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); if (pTable->schema == NULL) { mgmtDestroyNormalTable(pTable); - return NULL; + return -1; } - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); + memcpy(pTable->schema, pOper->rowData + tsNormalTableUpdateSize, schemaSize); pTable->sql = (char *)malloc(pTable->sqlLen); if (pTable->sql == NULL) { mgmtDestroyNormalTable(pTable); - return NULL; - } - memcpy(pTable->sql, str + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); - return (void *)pTable; -} - -void *mgmtNormalTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtNormalTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtNormalTableActionFp[(uint8_t)action]))(row, str, size, ssize); + return -1; } - return NULL; + memcpy(pTable->sql, pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); + + pOper->pObj = pTable; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitNormalTables() { @@ -218,12 +159,23 @@ int32_t mgmtInitNormalTables() { void *pLastNode = NULL; SNormalTableObj *pTable = NULL; - mgmtNormalTableActionInit(); SNormalTableObj tObj; - tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsNormalTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "ntables", + .hashSessions = TSDB_MAX_NORMAL_TABLES, + .maxRowSize = sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtNormalTableActionInsert, + .deleteFp = mgmtNormalTableActionDelete, + .updateFp = mgmtNormalTableActionUpdate, + .encodeFp = mgmtNormalTableActionEncode, + .decodeFp = mgmtNormalTableActionDecode, + .destroyFp = mgmtNormalTableActionDestroy, + }; - tsNormalTableSdb = sdbOpenTable(tsMaxTables, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction); + tsNormalTableSdb = sdbOpenTable(&tableDesc); if (tsNormalTableSdb == NULL) { mError("failed to init ntables data"); return -1; @@ -237,7 +189,11 @@ int32_t mgmtInitNormalTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ntable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -246,7 +202,11 @@ int32_t mgmtInitNormalTables() { if (pVgroup == NULL) { mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -255,7 +215,12 @@ int32_t mgmtInitNormalTables() { mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -263,19 +228,14 @@ int32_t mgmtInitNormalTables() { if (pVgroup->tableList == NULL) { mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } - - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *)pTable); - //pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); - - pTable->sql = (char *)pTable->schema + sizeof(SSchema) * pTable->numOfColumns; - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); } mTrace("ntables is initialized"); @@ -323,13 +283,6 @@ void *mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable) { } void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid) { - int32_t numOfTables = sdbGetNumOfRows(tsNormalTableSdb); - if (numOfTables >= TSDB_MAX_NORMAL_TABLES) { - mError("table:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_NORMAL_TABLES); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - SNormalTableObj *pTable = (SNormalTableObj *) calloc(sizeof(SNormalTableObj), 1); if (pTable == NULL) { mError("table:%s, failed to alloc memory", pCreate->tableId); @@ -341,7 +294,7 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->type = TSDB_NORMAL_TABLE; pTable->vgId = pVgroup->vgId; pTable->createdTime = taosGetTimestampMs(); - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = sid; pTable->sversion = 0; pTable->numOfColumns = htons(pCreate->numOfColumns); @@ -377,7 +330,11 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql); } - if (sdbInsertRow(tsNormalTableSdb, pTable, 0) < 0) { + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + if (sdbInsertRow(&desc) < 0) { mError("table:%s, update sdb error", pTable->tableId); free(pTable); terrno = TSDB_CODE_SDB_ERROR; @@ -474,7 +431,14 @@ int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int3 pTable->sversion++; pAcct->acctInfo.numOfTimeSeries += ncols; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsNormalTableUpdateSize; + sdbUpdateRow(&desc); + return TSDB_CODE_SUCCESS; } @@ -503,7 +467,14 @@ int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) pTable->sversion++; pAcct->acctInfo.numOfTimeSeries--; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsNormalTableUpdateSize; + sdbUpdateRow(&desc); return TSDB_CODE_SUCCESS; } @@ -564,12 +535,17 @@ void mgmtDropAllNormalTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsNormalTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; continue; } } - mTrace("db:%s, all normal tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all normal tables:%d is dropped from sdb", pDropDb->name, numOfTables); } diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index b20e66dd874b6b589087b37e66f463e5d32aee89..1b98ac95966a8c832eabe0beff7fcbd519d97bea 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -762,3 +762,13 @@ int32_t mgmtInitProfile() { void mgmtCleanUpProfile() { } + +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) { + if (pMsg != NULL) { + if (pMsg->pCont != NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + free(pMsg); + } +} \ No newline at end of file diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c new file mode 100644 index 0000000000000000000000000000000000000000..0b1ac3be9706ea8c7342914a4fa87e06ca7cb94c --- /dev/null +++ b/src/mnode/src/mgmtSdb.c @@ -0,0 +1,767 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tchecksum.h" +#include "tglobalcfg.h" +#include "tlog.h" +#include "trpc.h" +#include "tutil.h" +#include "hashint.h" +#include "hashstr.h" +#include "mgmtSdb.h" + +#define abs(x) (((x) < 0) ? -(x) : (x)) +#define SDB_MAX_PEERS 4 +#define SDB_DELIMITER 0xFFF00F00 +#define SDB_ENDCOMMIT 0xAFFFAAAF + +typedef struct { + uint64_t swVersion; + int16_t sdbFileVersion; + char reserved[2]; + TSCKSUM checkSum; +} SSdbHeader; + +typedef struct _SSdbTable { + SSdbHeader header; + char tableName[TSDB_DB_NAME_LEN]; + char fileName[TSDB_FILENAME_LEN]; + ESdbKeyType keyType; + int32_t tableId; + int32_t hashSessions; + int32_t maxRowSize; + int32_t autoIndex; + int32_t fd; + int64_t numOfRows; + int64_t version; + int64_t fileSize; + void * iHandle; + int32_t (*insertFp)(SSdbOperDesc *pDesc); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*destroyFp)(SSdbOperDesc *pOper); + pthread_mutex_t mutex; +} SSdbTable; + +typedef struct { + int64_t version; + int64_t offset; + int32_t rowSize; + void * row; +} SRowMeta; + +typedef struct { + int32_t delimiter; + int32_t rowSize; + int64_t version; + char data[]; +} SRowHead; + +typedef enum { + SDB_FORWARD_TYPE_INSERT, + SDB_FORWARD_TYPE_DELETE, + SDB_FORWARD_TYPE_UPDATE +} ESdbForwardType; + +typedef struct { + ESdbForwardType type; + int32_t tableId; + int64_t version; + int32_t rowSize; + void * rowData; +} SForwardMsg; + +extern char version[]; +const int16_t sdbFileVersion = 2; +int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL; + +static SSdbTable *sdbTableList[10] = {0}; +static int32_t sdbNumOfTables = 0; +static uint64_t sdbVersion = 0; + +static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; +static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; +static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; +static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData}; +static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; +static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; + +uint64_t sdbGetVersion() { return sdbVersion; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } +int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } + +static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { + static char str[16]; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + return (char *)row; + case SDB_KEY_TYPE_AUTO: + sprintf(str, "%d", *(int32_t *)row); + return str; + default: + return "unknown"; + } +} + +static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) { + if (mpeerForwardRequestFp) { + return mpeerForwardRequestFp(forwardMsg); + } else { + return 0; + } +} + +static void sdbFinishCommit(SSdbTable *pTable) { + uint32_t sdbEcommit = SDB_ENDCOMMIT; + off_t offset = lseek(pTable->fd, 0, SEEK_END); + assert(offset == pTable->fileSize); + twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); + pTable->fileSize += sizeof(sdbEcommit); +} + +static int32_t sdbOpenSdbFile(SSdbTable *pTable) { + struct stat fstat, ofstat; + uint64_t size; + char * dirc = NULL; + char * basec = NULL; + union { + char cversion[64]; + uint64_t iversion; + } swVersion; + + memcpy(swVersion.cversion, version, sizeof(uint64_t)); + + // check sdb.db and .sdb.db status + char fn[TSDB_FILENAME_LEN] = "\0"; + dirc = strdup(pTable->fileName); + basec = strdup(pTable->fileName); + sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); + tfree(dirc); + tfree(basec); + if (stat(fn, &ofstat) == 0) { // .sdb.db file exists + if (stat(pTable->fileName, &fstat) == 0) { + remove(fn); + } else { + remove(pTable->fileName); + rename(fn, pTable->fileName); + } + } + + pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + if (pTable->fd < 0) { + sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName); + return -1; + } + + pTable->fileSize = 0; + stat(pTable->fileName, &fstat); + size = sizeof(pTable->header); + + if (fstat.st_size == 0) { + pTable->header.swVersion = swVersion.iversion; + pTable->header.sdbFileVersion = sdbFileVersion; + if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { + sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + twrite(pTable->fd, &(pTable->header), size); + pTable->fileSize += size; + sdbFinishCommit(pTable); + } else { + uint32_t sdbEcommit = 0; + off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END); + while (offset > 0) { + read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); + if (sdbEcommit == SDB_ENDCOMMIT) { + ftruncate(pTable->fd, offset + sizeof(sdbEcommit)); + break; + } + offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR); + } + lseek(pTable->fd, 0, SEEK_SET); + + ssize_t tsize = read(pTable->fd, &(pTable->header), size); + if (tsize < size) { + sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + + if (pTable->header.swVersion != swVersion.iversion) { + sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName); + } + + if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { + sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + + pTable->fileSize += size; + // skip end commit symbol + lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); + pTable->fileSize += sizeof(sdbEcommit); + } + + pTable->numOfRows = 0; + + return pTable->fd; +} + +static int32_t sdbInitTableByFile(SSdbTable *pTable) { + sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName); + if (sdbOpenSdbFile(pTable) < 0) { + sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName); + return -1; + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)malloc(total_size); + if (rowHead == NULL) { + sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName); + return -1; + } + + int32_t numOfChanged = 0; + int32_t maxAutoIndex = 0; + while (1) { + memset(rowHead, 0, total_size); + + int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); + if (bytes < 0) { + sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName); + tfree(rowHead); + return -1; + } + + if (bytes == 0) break; + + if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { + pTable->fileSize++; + lseek(pTable->fd, -(bytes - 1), SEEK_CUR); + continue; + } + + if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { + sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName, + pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize); + pTable->fileSize += sizeof(SRowHead); + continue; + } + + bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); + if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { + // TODO: Here may cause pTable->fileSize not end of the file + sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version, + rowHead->rowSize); + break; + } + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { + sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version); + pTable->fileSize += real_size; + continue; + } + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); + } + + pTable->version = MAX(pTable->version, abs(rowHead->version)); + + void *pMetaRow = sdbGetRow(pTable, rowHead->data); + if (pMetaRow == NULL) { + if (rowHead->version < 0) { + sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version, + sdbGetkeyStr(pTable, rowHead->data)); + } else { + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize + }; + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + pTable->numOfRows++; + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } + } + } else { + if (rowHead->version < 0) { + SSdbOperDesc oper = { + .table = pTable, + .pObj = pMetaRow + }; + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + pTable->numOfRows--; + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize, + .pObj = pMetaRow + }; + + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } + } + numOfChanged++; + } + + pTable->fileSize += real_size; + pTable->fileSize += 4; + lseek(pTable->fd, 4, SEEK_CUR); + } + + void *pNode = NULL; + while (1) { + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + .version = pMeta->version, + }; + + int32_t code = (*pTable->insertFp)(&oper); + if (code != TSDB_CODE_SUCCESS) { + sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data)); + } + } + + sdbVersion += pTable->version; + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + pTable->autoIndex = maxAutoIndex; + } + + tfree(rowHead); + return 0; +} + +void *sdbOpenTable(SSdbTableDesc *pDesc) { + SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); + if (pTable == NULL) return NULL; + + pTable->keyType = pDesc->keyType; + pTable->hashSessions = pDesc->hashSessions; + pTable->maxRowSize = pDesc->maxRowSize; + pTable->insertFp = pDesc->insertFp; + pTable->deleteFp = pDesc->deleteFp; + pTable->updateFp = pDesc->updateFp; + pTable->encodeFp = pDesc->encodeFp; + pTable->decodeFp = pDesc->decodeFp; + pTable->destroyFp = pDesc->destroyFp; + strcpy(pTable->tableName, pDesc->tableName); + sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName); + + if (sdbInitIndexFp[pTable->keyType] != NULL) { + pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); + } + + pthread_mutex_init(&pTable->mutex, NULL); + + if (sdbInitTableByFile(pTable) < 0) return NULL; + + pTable->tableId = sdbNumOfTables++; + sdbTableList[pTable->tableId] = pTable; + + sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d", + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables); + + return pTable; +} + +static SRowMeta *sdbGetRowMeta(void *handle, void *key) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + if (handle == NULL) return NULL; + + pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); + + return pMeta; +} + +void *sdbGetRow(void *handle, void *key) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + if (handle == NULL) return NULL; + + pthread_mutex_lock(&pTable->mutex); + pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); + pthread_mutex_unlock(&pTable->mutex); + + if (pMeta == NULL) { + return NULL; + } + + return pMeta->row; +} + +int32_t sdbInsertRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) { + sdbError("sdb tables is null"); + return TSDB_CODE_OTHERS; + } + + if (sdbGetRow(pTable, pOper->pObj)) { + sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); + return TSDB_CODE_ALREADY_THERE; + } + + pOper->maxRowSize = pTable->maxRowSize; + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_INSERT, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); + pthread_mutex_unlock(&pTable->mutex); + return TSDB_CODE_OTHERS; + } + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + pthread_mutex_unlock(&pTable->mutex); + sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); + return -1; + } + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + } + pTable->version++; + sdbVersion++; + + pOper->rowData = rowHead->data; + (*pTable->encodeFp)(pOper); + rowHead->rowSize = pOper->rowSize; + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + tfree(rowHead); + + // update in SDB layer + SRowMeta rowMeta; + rowMeta.version = pTable->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = pOper->rowSize; + rowMeta.row = pOper->pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); + + pTable->numOfRows++; + + pthread_mutex_unlock(&pTable->mutex); + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + (*pTable->insertFp)(pOper); + + return 0; +} + +// row here can be object or null-terminated string +int32_t sdbDeleteRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) return -1; + + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + if (pMeta == NULL) { + sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); + return -1; + } + + void * pMetaRow = pMeta->row; + assert(pMetaRow != NULL); + + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_DELETE, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pMeta->rowSize, + .rowData = pMeta->row, + }; + + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to delete record", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + } + + int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + + pTable->version++; + sdbVersion++; + + int32_t rowSize = 0; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + rowSize = strlen((char *)pOper->pObj) + 1; + break; + case SDB_KEY_TYPE_AUTO: + rowSize = sizeof(uint64_t); + break; + default: + return -1; + } + + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->version = -(pTable->version); + memcpy(rowHead->data, pOper->pObj, rowSize); + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + + tfree(rowHead); + + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + // Delete from current layer + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); + + pTable->numOfRows--; + + pthread_mutex_unlock(&pTable->mutex); + + (*pTable->deleteFp)(pOper); + (*pTable->destroyFp)(pOper); + + return 0; +} + +// row here can be the object or the string info (encoded string) +int32_t sdbUpdateRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) return -1; + + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + if (pMeta == NULL) { + sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); + return -1; + } + + void *pMetaRow = pMeta->row; + assert(pMetaRow != NULL); + + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_UPDATE, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to update record", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("table:%s, failed to allocate row head memory", pTable->tableName); + return -1; + } + + if (pMetaRow != pOper->pObj) { + memcpy(rowHead->data, pOper->rowData, pOper->rowSize); + rowHead->rowSize = pOper->rowSize; + } else { + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .maxRowSize = pTable->maxRowSize, + .pObj = pOper->pObj + }; + (*pTable->encodeFp)(&oper); + rowHead->rowSize = oper.rowSize; + } + + pTable->version++; + sdbVersion++; + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + pMeta->version = pTable->version; + pMeta->offset = pTable->fileSize; + pMeta->rowSize = rowHead->rowSize; + + pthread_mutex_unlock(&pTable->mutex); + + (*pTable->updateFp)(pOper); // update in upper layer + + tfree(rowHead); + + return 0; +} + +void sdbCloseTable(void *handle) { + SSdbTable *pTable = (SSdbTable *)handle; + void * pNode = NULL; + + if (pTable == NULL) return; + + while (1) { + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + .version = pMeta->version, + }; + + (*pTable->destroyFp)(&oper); + } + + if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); + + if (pTable->fd) tclose(pTable->fd); + + pthread_mutex_destroy(&pTable->mutex); + + sdbNumOfTables--; + sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); + + tfree(pTable); +} + +void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + *ppRow = NULL; + if (pTable == NULL) return NULL; + + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) return NULL; + + *ppRow = pMeta->row; + + return pNode; +} diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 9cf0e3b57338f113996579d9cb129013691fdea9..3ff78b985e4cb55d07d3284f750a56bc4fb99faf 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -32,6 +32,7 @@ #include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" @@ -63,7 +64,7 @@ int32_t mgmtInitShell() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg); - tsMgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT"); + tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT"); int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0; if (numOfThreads < 1) { @@ -131,7 +132,7 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { } static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { - if (sdbGetRunStatus() != SDB_STATUS_SERVING) { + if (!mgmtInServerStatus()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; @@ -309,20 +310,10 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { return; } - pHBRsp->ipList.inUse = 0; - pHBRsp->ipList.port = htons(tsMnodeShellPort); - pHBRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pHBRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pHBRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pHBRsp->ipList); } /* @@ -411,20 +402,11 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { strcpy(pConnectRsp->serverVersion, version); pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - pConnectRsp->ipList.inUse = 0; - pConnectRsp->ipList.port = htons(tsMnodeShellPort); - pConnectRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pConnectRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pConnectRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pConnectRsp->ipList); } connect_over: diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index bd697ade93288917e38bb1388049d98717bdd9c3..f2b54ec000091914f49a5f1b94b97cffc1bed8e6 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -23,6 +23,7 @@ #include "mgmtGrant.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" +#include "mgmtSdb.h" #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" @@ -31,160 +32,106 @@ #include "name.h" #include "tsqlfunction.h" -static void *tsSuperTableSdb; +static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; -static void *(*mgmtSuperTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static void mgmtDestroySuperTable(SSuperTableObj *pTable) { - free(pTable->schema); - free(pTable); +static void mgmtDestroySuperTable(SSuperTableObj *pStable) { + tfree(pStable->schema); + tfree(pStable); } -static void mgmtSuperTableActionInit() { - SSuperTableObj tObj; - tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtSuperTableActionFp[SDB_TYPE_INSERT] = mgmtSuperTableActionInsert; - mgmtSuperTableActionFp[SDB_TYPE_DELETE] = mgmtSuperTableActionDelete; - mgmtSuperTableActionFp[SDB_TYPE_UPDATE] = mgmtSuperTableActionUpdate; - mgmtSuperTableActionFp[SDB_TYPE_ENCODE] = mgmtSuperTableActionEncode; - mgmtSuperTableActionFp[SDB_TYPE_DECODE] = mgmtSuperTableActionDecode; - mgmtSuperTableActionFp[SDB_TYPE_RESET] = mgmtSuperTableActionReset; - mgmtSuperTableActionFp[SDB_TYPE_DESTROY] = mgmtSuperTableActionDestroy; -} - -void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - memcpy(pTable, str, tsSuperTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = realloc(pTable->schema, schemaSize); - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); - - return NULL; -} - -void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - mgmtDestroySuperTable(pTable); - return NULL; +static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroySuperTable(pOper->pObj); + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { + STableInfo *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + if (pDb != NULL) { mgmtAddSuperTableIntoDb(pDb); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) { + STableInfo *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + if (pDb != NULL) { mgmtRemoveSuperTableFromDb(pDb); + mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtSuperTableActionReset(row, str, size, NULL); +static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) { + SSuperTableObj *pStable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - if (size < tsSuperTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(str, pTable, tsSuperTableUpdateSize); - memcpy(str + tsSuperTableUpdateSize, pTable->schema, schemaSize); - *ssize = tsSuperTableUpdateSize + schemaSize; + memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize); + memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize); + pOper->rowSize = tsSuperTableUpdateSize + schemaSize; - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SSuperTableObj *pTable = (SSuperTableObj *) malloc(sizeof(SSuperTableObj)); - if (pTable == NULL) { - return NULL; - } - memset(pTable, 0, sizeof(SSuperTableObj)); + SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); + if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - if (size < tsSuperTableUpdateSize) { - mgmtDestroySuperTable(pTable); - return NULL; - } - memcpy(pTable, str, tsSuperTableUpdateSize); + memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize); - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = malloc(schemaSize); - if (pTable->schema == NULL) { - mgmtDestroySuperTable(pTable); - return NULL; + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); + pStable->schema = malloc(schemaSize); + if (pStable->schema == NULL) { + mgmtDestroySuperTable(pStable); + return -1; } - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); - return (void *) pTable; -} + memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize); + pOper->pObj = pStable; -void *mgmtSuperTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtSuperTableActionFp[(uint8_t) action] != NULL) { - return (*(mgmtSuperTableActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitSuperTables() { - void *pNode = NULL; - void *pLastNode = NULL; - SSuperTableObj *pTable = NULL; - - mgmtSuperTableActionInit(); - - tsSuperTableSdb = sdbOpenTable(tsMaxTables, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction); + SSuperTableObj tObj; + tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "stables", + .hashSessions = TSDB_MAX_SUPER_TABLES, + .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtSuperTableActionInsert, + .deleteFp = mgmtSuperTableActionDelete, + .updateFp = mgmtSuperTableActionUpdate, + .encodeFp = mgmtSuperTableActionEncode, + .decodeFp = mgmtSuperTableActionDecode, + .destroyFp = mgmtSuperTableActionDestroy, + }; + + tsSuperTableSdb = sdbOpenTable(&tableDesc); if (tsSuperTableSdb == NULL) { mError("failed to init stables data"); return -1; } - pNode = NULL; - while (1) { - pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **) &pTable); - if (pTable == NULL) { - break; - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("super table:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsSuperTableSdb, pTable); - pNode = pLastNode; - continue; - } - - mgmtAddSuperTableIntoDb(pDb); - } - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mgmtGetShowSuperTableMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mgmtRetrieveShowSuperTables); @@ -197,13 +144,7 @@ void mgmtCleanUpSuperTables() { } int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { - int32_t numOfTables = sdbGetNumOfRows(tsSuperTableSdb); - if (numOfTables >= TSDB_MAX_SUPER_TABLES) { - mError("stable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_SUPER_TABLES); - return TSDB_CODE_TOO_MANY_TABLES; - } - - SSuperTableObj *pStable = (SSuperTableObj *)calloc(sizeof(SSuperTableObj), 1); + SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -213,7 +154,7 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { pStable->createdTime = taosGetTimestampMs(); pStable->vgId = 0; pStable->sid = 0; - pStable->uid = (((uint64_t) pStable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pStable->sversion = 0; pStable->numOfColumns = htons(pCreate->numOfColumns); pStable->numOfTags = htons(pCreate->numOfTags); @@ -235,13 +176,21 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { tschema[col].bytes = htons(tschema[col].bytes); } - if (sdbInsertRow(tsSuperTableSdb, pStable, 0) < 0) { - mError("stable:%s, update sdb error", pStable->tableId); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable, + .rowSize = sizeof(SSuperTableObj) + schemaSize + }; + + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mgmtDestroySuperTable(pStable); return TSDB_CODE_SDB_ERROR; + } else { + mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); + return TSDB_CODE_SUCCESS; } - - mPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); - return TSDB_CODE_SUCCESS; } int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pStable) { @@ -249,10 +198,14 @@ int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pSta mError("stable:%s, numOfTables:%d not 0", pStable->tableId, pStable->numOfTables); return TSDB_CODE_OTHERS; } else { - //TODO: drop child tables - mError("stable:%s, is dropped from sdb", pStable->tableId); - mgmtRemoveSuperTableFromDb(pDb); - return TSDB_CODE_OTHERS; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable + }; + int32_t code = sdbDeleteRow(&oper); + mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->tableId, tstrerror(code)); + return code; } } @@ -261,10 +214,9 @@ void* mgmtGetSuperTable(char *tableId) { } void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { - //TODO get vgroup of dnodes SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum()); - rsp->numOfDnodes = 1; - rsp->dnodeIps[0] = 0; + rsp->numOfDnodes = htonl(1); + rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp)); return rsp; } @@ -321,7 +273,7 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId); return TSDB_CODE_SUCCESS; @@ -354,7 +306,7 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -367,8 +319,8 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN return TSDB_CODE_INVALID_MSG_TYPE; } - int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + int32_t rowSize = 0; + uint32_t len = strlen(newTagName); if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { return TSDB_CODE_APP_ERROR; @@ -384,9 +336,10 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN if (msg == NULL) return TSDB_CODE_APP_ERROR; memset(msg, 0, size); - mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); + // mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); - int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, rowSize, 1); + int32_t ret = 0; + // int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); if (ret < 0) { @@ -448,7 +401,7 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -481,7 +434,7 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { pStable->schema = realloc(pStable->schema, schemaSize); pAcct->acctInfo.numOfTimeSeries -= (pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -620,14 +573,19 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsSuperTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsSuperTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; numOfTables ++; continue; } } - mTrace("db:%s, all super tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable) { diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 2fe6e31306ecb49e81f31b6fab8c3d93843ccab5..675de57924ccb46650085510079459a6c678f454 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -22,10 +22,12 @@ #include "mgmtDClient.h" #include "mgmtDb.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtGrant.h" #include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtUser.h" @@ -60,6 +62,7 @@ static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle); +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg); int32_t mgmtInitTables() { int32_t code = mgmtInitSuperTables(); @@ -77,8 +80,6 @@ int32_t mgmtInitTables() { return code; } - mgmtSetVgroupIdPool(); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_TABLE, mgmtProcessCreateTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_TABLE, mgmtProcessDropTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mgmtProcessAlterTableMsg); @@ -91,6 +92,7 @@ int32_t mgmtInitTables() { mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg); return TSDB_CODE_SUCCESS; } @@ -201,13 +203,13 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "table_name"); + strcpy(pSchema[cols].name, "table name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -219,7 +221,7 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "stable_name"); + strcpy(pSchema[cols].name, "stable name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -237,9 +239,6 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -/* - * remove the hole in result set - */ static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { if (rows < capacity) { for (int32_t i = 0; i < numOfCols; ++i) { @@ -433,11 +432,13 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { pTable = mgmtCreateChildTable(pCreate, pVgroup, sid); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable); if (pMDCreate == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } } else { @@ -445,11 +446,13 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { pTable = mgmtCreateNormalTable(pCreate, pVgroup, sid); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); if (pMDCreate == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } } @@ -760,9 +763,19 @@ static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (pTable->type == TSDB_CHILD_TABLE) { - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); } else if (pTable->type == TSDB_NORMAL_TABLE){ - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsNormalTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); } else {} mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); @@ -816,14 +829,26 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { } if (pTable->type == TSDB_CHILD_TABLE) { - if (sdbDeleteRow(tsChildTableSdb, pTable) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { mError("table:%s, update ctables sdb error", pTable->tableId); mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); free(queueMsg); return; } } else if (pTable->type == TSDB_NORMAL_TABLE){ - if (sdbDeleteRow(tsNormalTableSdb, pTable) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsNormalTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { mError("table:%s, update ntables sdb error", pTable->tableId); mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); free(queueMsg); @@ -844,32 +869,48 @@ static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); } -// -// -//static void mgmtProcessTableCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *thandle) { -// SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// pCfg->sid = htonl(pCfg->sid); -// mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// -// if (!sdbMaster) { -// mError("dnode:%s, vnode:%d, sid:%d, not master, redirect it", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); -// if (pTable == NULL) { -// mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_INVALID_TABLE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// //TODO -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateTableMsg(NULL, &ipSet, NULL); -//} -// \ No newline at end of file +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + pCfg->sid = htonl(pCfg->sid); + mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + + STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); + if (pTable == NULL) { + mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SMDCreateTableMsg *pMDCreate = NULL; + if (pTable->type == TSDB_CHILD_TABLE) { + mTrace("table:%s, is a child table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + pMDCreate = mgmtBuildCreateChildTableMsg(NULL, (SChildTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + } else if (pTable->type == TSDB_NORMAL_TABLE) { + mTrace("table:%s, is a normal table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + } else { + mError("table:%s, invalid msg type, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + } + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + SRpcMsg rpcRsp = { + .handle = NULL, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + mgmtSendMsgToDnode(&ipSet, &rpcRsp); +} diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 771c78c2229a2191cb64973a81553b026e38926f..576b326669e6bdd8e29e505cdd350a31147ccf6d 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -17,9 +17,11 @@ #include "os.h" #include "trpc.h" #include "ttime.h" +#include "tutil.h" #include "mgmtAcct.h" #include "mgmtGrant.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" @@ -36,48 +38,84 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); -static void *(*mgmtUserActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize); -static void mgmtUserActionInit(); +static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; +} -int32_t mgmtInitUsers() { - void *pNode = NULL; - SUserObj *pUser = NULL; - SAcctObj *pAcct = NULL; - int32_t numOfUsers = 0; +static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - mgmtUserActionInit(); + if (pAcct != NULL) { + mgmtAddUserIntoAcct(pAcct, pUser); + } + else { + mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct); + return TSDB_CODE_INVALID_ACCT; + } - SUserObj tObj; - tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + return TSDB_CODE_SUCCESS; +} - tsUserSdb = sdbOpenTable(tsMaxUsers, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); - if (tsUserSdb == NULL) { - mError("failed to init user data"); +static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); + + mgmtRemoveUserFromAcct(pAcct, pUser); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + + if (pOper->maxRowSize < tsUserUpdateSize) { return -1; + } else { + memcpy(pOper->rowData, pUser, tsUserUpdateSize); + pOper->rowSize = tsUserUpdateSize; + return TSDB_CODE_SUCCESS; } +} - while (1) { - pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser); - if (pUser == NULL) break; - - pUser->prev = NULL; - pUser->next = NULL; +static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { + SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj)); + if (pUser == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - pAcct = mgmtGetAcct(pUser->acct); - mgmtAddUserIntoAcct(pAcct, pUser); + memcpy(pUser, pOper->rowData, tsUserUpdateSize); + pOper->pObj = pUser; + return TSDB_CODE_SUCCESS; +} - numOfUsers++; +int32_t mgmtInitUsers() { + SUserObj tObj; + tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "users", + .hashSessions = TSDB_MAX_USERS, + .maxRowSize = tsUserUpdateSize, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtUserActionInsert, + .deleteFp = mgmtUserActionDelete, + .updateFp = mgmtUserActionUpdate, + .encodeFp = mgmtUserActionEncode, + .decodeFp = mgmtUserActionDecode, + .destroyFp = mgmtUserActionDestroy, + }; + + tsUserSdb = sdbOpenTable(&tableDesc); + if (tsUserSdb == NULL) { + mError("failed to init user data"); + return -1; } - pAcct = mgmtGetAcct("root"); + SAcctObj *pAcct = mgmtGetAcct("root"); mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "monitor", tsInternalPass); mgmtCreateUser(pAcct, "_root", tsInternalPass); @@ -101,16 +139,23 @@ SUserObj *mgmtGetUser(char *name) { } static int32_t mgmtUpdateUser(SUserObj *pUser) { - return sdbUpdateRow(tsUserSdb, pUser, 0, 1); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = tsUserUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pUser); + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { - int32_t numOfUsers = sdbGetNumOfRows(tsUserSdb); - if (numOfUsers >= tsMaxUsers) { - mWarn("numOfUsers:%d, exceed tsMaxUsers:%d", numOfUsers, tsMaxUsers); - return TSDB_CODE_TOO_MANY_USERS; - } - int32_t code = mgmtCheckUserLimit(pAcct); if (code != 0) { return code; @@ -131,8 +176,7 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { return code; } - pUser = malloc(sizeof(SUserObj)); - memset(pUser, 0, sizeof(SUserObj)); + pUser = calloc(1, sizeof(SUserObj)); strcpy(pUser->user, name); taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass); strcpy(pUser->acct, pAcct->user); @@ -143,8 +187,15 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { pUser->superAuth = 1; } - code = TSDB_CODE_SUCCESS; - if (sdbInsertRow(tsUserSdb, pUser, 0) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pUser); code = TSDB_CODE_SDB_ERROR; } @@ -165,9 +216,18 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { return TSDB_CODE_NO_RIGHTS; } - sdbDeleteRow(tsUserSdb, pUser); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser + }; - return 0; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { @@ -250,83 +310,6 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void return numOfRows; } -static void mgmtUserActionInit() { - mgmtUserActionFp[SDB_TYPE_INSERT] = mgmtUserActionInsert; - mgmtUserActionFp[SDB_TYPE_DELETE] = mgmtUserActionDelete; - mgmtUserActionFp[SDB_TYPE_UPDATE] = mgmtUserActionUpdate; - mgmtUserActionFp[SDB_TYPE_ENCODE] = mgmtUserActionEncode; - mgmtUserActionFp[SDB_TYPE_DECODE] = mgmtUserActionDecode; - mgmtUserActionFp[SDB_TYPE_RESET] = mgmtUserActionReset; - mgmtUserActionFp[SDB_TYPE_DESTROY] = mgmtUserActionDestroy; -} - -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtUserActionFp[(uint8_t) action] != NULL) { - return (*(mgmtUserActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; -} - -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - pUser->pAcct = pAcct; - mgmtAddUserIntoAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - mgmtRemoveUserFromAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtUserActionReset(row, str, size, ssize); -} - -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - - if (size < tsUserUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pUser, tsUserUpdateSize); - *ssize = tsUserUpdateSize; - } - - return NULL; -} - -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); - if (pUser == NULL) return NULL; - memset(pUser, 0, sizeof(SUserObj)); - - memcpy(pUser, str, tsUserUpdateSize); - - return (void *)pUser; -} - -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *)row; - - memcpy(pUser, str, tsUserUpdateSize); - - return NULL; -} - -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - - return NULL; -} - SUserObj *mgmtGetUserFromConn(void *pConn) { SRpcConnInfo connInfo; if (rpcGetConnInfo(pConn, &connInfo) == 0) { @@ -393,7 +376,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { memset(pUser->pass, 0, sizeof(pUser->pass)); taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass); code = mgmtUpdateUser(pUser); - mLPrint("user:%s password is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s password is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } @@ -428,10 +411,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } if (hasRight) { - //if (pAlter->privilege == 1) { // super - // pUser->superAuth = 1; - // pUser->writeAuth = 1; - //} if (pAlter->privilege == 2) { // read pUser->superAuth = 0; pUser->writeAuth = 0; @@ -442,7 +421,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } code = mgmtUpdateUser(pUser); - mLPrint("user:%s privilege is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s privilege is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } @@ -492,7 +471,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { if (hasRight) { code = mgmtDropUser(pUser->pAcct, pDrop->user); if (code == TSDB_CODE_SUCCESS) { - mLPrint("user:%s is dropped by %s", pDrop->user, pUser->user); + mLPrint("user:%s is dropped by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } } else { code = TSDB_CODE_NO_RIGHTS; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index f84b5443a5bd289c78bf66f3672fc4626fe722d9..9e52574b8c8890289f12770229675e99d46cb749 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -23,104 +23,151 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtTable.h" #include "mgmtVgroup.h" -static void *tsVgroupSdb = NULL; +static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; -static void *(*mgmtVgroupActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg); +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) ; static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); -static void mgmtVgroupActionInit() { - SVgObj tObj; - tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtVgroupActionFp[SDB_TYPE_INSERT] = mgmtVgroupActionInsert; - mgmtVgroupActionFp[SDB_TYPE_DELETE] = mgmtVgroupActionDelete; - mgmtVgroupActionFp[SDB_TYPE_UPDATE] = mgmtVgroupActionUpdate; - mgmtVgroupActionFp[SDB_TYPE_ENCODE] = mgmtVgroupActionEncode; - mgmtVgroupActionFp[SDB_TYPE_DECODE] = mgmtVgroupActionDecode; - mgmtVgroupActionFp[SDB_TYPE_RESET] = mgmtVgroupActionReset; - mgmtVgroupActionFp[SDB_TYPE_DESTROY] = mgmtVgroupActionDestroy; +static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + if (pVgroup->idPool) { + taosIdPoolCleanUp(pVgroup->idPool); + pVgroup->idPool = NULL; + } + if (pVgroup->tableList) { + tfree(pVgroup->tableList); + } + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtVgroupAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtVgroupActionFp[(uint8_t) action] != NULL) { - return (*(mgmtVgroupActionFp[(uint8_t) action]))(row, str, size, ssize); +static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + if (pDb == NULL) { + return TSDB_CODE_INVALID_DB; } - return NULL; -} -int32_t mgmtInitVgroups() { - void *pNode = NULL; - SVgObj *pVgroup = NULL; + pVgroup->pDb = pDb; + pVgroup->prev = NULL; + pVgroup->next = NULL; - mgmtVgroupActionInit(); + int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; + pVgroup->tableList = (STableInfo **)calloc(pDb->cfg.maxSessions, sizeof(STableInfo *)); + if (pVgroup->tableList == NULL) { + mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); + return -1; + } - tsVgroupSdb = sdbOpenTable(tsMaxVGroups, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); - if (tsVgroupSdb == NULL) { - mError("failed to init vgroups data"); + pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + if (pVgroup->idPool == NULL) { + mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId); + tfree(pVgroup->tableList); return -1; } - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL) break; + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + pVgroup->vnodeGid[i].privateIp = pDnode->privateIp; + pVgroup->vnodeGid[i].publicIp = pDnode->publicIp; + pVgroup->vnodeGid[i].vnode = pVgroup->vgId; + } - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) continue; + mgmtAddVgroupIntoDb(pVgroup); - pVgroup->prev = NULL; - pVgroup->next = NULL; + return TSDB_CODE_SUCCESS; +} - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)malloc(size); - if (pVgroup->tableList == NULL) { - mError("failed to malloc(size:%d) for the tableList of vgroups", size); - return -1; - } - - memset(pVgroup->tableList, 0, size); +static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + + if (pVgroup->pDb != NULL) { + mgmtRemoveVgroupFromDb(pVgroup); + } - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); - if (pVgroup->idPool == NULL) { - mError("failed to taosInitIdPool for vgroups"); - free(pVgroup->tableList); - return -1; - } - - taosIdPoolReinit(pVgroup->idPool); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - if (tsIsCluster && pVgroup->vnodeGid[0].publicIp == 0) { - pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); - pVgroup->vnodeGid[0].privateIp = inet_addr(tsPrivateIp); - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); + SDbObj *pDb = pVgroup->pDb; + if (pDb != NULL) { + if (pDb->cfg.maxSessions != oldTables) { + mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); + taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); + int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; + pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); } + } + + mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxSessions, pVgroup->numOfVnodes); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + if (pOper->maxRowSize < tsVgUpdateSize) { + return -1; + } else { + memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); + pOper->rowSize = tsVgUpdateSize; + return TSDB_CODE_SUCCESS; + } +} + +static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj)); + if (pVgroup == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); + memcpy(pVgroup, pOper->rowData, tsVgUpdateSize); + pOper->pObj = pVgroup; + return TSDB_CODE_SUCCESS; +} + +int32_t mgmtInitVgroups() { + SVgObj tObj; + tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "vgroups", + .hashSessions = TSDB_MAX_VGROUPS, + .maxRowSize = tsVgUpdateSize, + .keyType = SDB_KEY_TYPE_AUTO, + .insertFp = mgmtVgroupActionInsert, + .deleteFp = mgmtVgroupActionDelete, + .updateFp = mgmtVgroupActionUpdate, + .encodeFp = mgmtVgroupActionEncode, + .decodeFp = mgmtVgroupActionDecode, + .destroyFp = mgmtVgroupActionDestroy, + }; + + tsVgroupSdb = sdbOpenTable(&tableDesc); + if (tsVgroupSdb == NULL) { + mError("failed to init vgroups data"); + return -1; } mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mgmtGetVgroupMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mgmtRetrieveVgroups); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg); mTrace("vgroup is initialized"); return 0; @@ -139,32 +186,41 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg) { if (pDb == NULL) { mError("failed to create vgroup, db not found"); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + mgmtFreeQueuedMsg(pMsg); return; } - SVgObj *pVgroup = (SVgObj *)calloc(sizeof(SVgObj), 1); + SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); strcpy(pVgroup->dbName, pDb->name); pVgroup->numOfVnodes = pDb->cfg.replications; + pVgroup->createdTime = taosGetTimestampMs(); if (mgmtAllocVnodes(pVgroup) != 0) { mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes); free(pVgroup); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES); + mgmtFreeQueuedMsg(pMsg); return; } - pVgroup->createdTime = taosGetTimestampMs(); - pVgroup->tableList = (STableInfo **) calloc(sizeof(STableInfo *), pDb->cfg.maxSessions); - pVgroup->numOfTables = 0; - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); - - mgmtAddVgroupIntoDb(pDb, pVgroup); - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj) + }; - sdbInsertRow(tsVgroupSdb, pVgroup, 0); + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pVgroup); + code = TSDB_CODE_SDB_ERROR; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); + mgmtFreeQueuedMsg(pMsg); + return; + } mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - mPrint("vgroup:%d, dnode:%d vnode:%d", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); + mPrint("vgroup:%d, index:%d, dnode:%d vnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); } pMsg->ahandle = pVgroup; @@ -178,28 +234,12 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); - sdbDeleteRow(tsVgroupSdb, pVgroup); - } -} - -void mgmtSetVgroupIdPool() { - void * pNode = NULL; - SVgObj *pVgroup = NULL; - SDbObj *pDb; - - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL || pVgroup->idPool == 0) break; - - taosIdPoolSetFreeList(pVgroup->idPool); - pVgroup->numOfTables = taosIdPoolNumOfUsed(pVgroup->idPool); - - pDb = mgmtGetDb(pVgroup->dbName); - pDb->numOfTables += pVgroup->numOfTables; - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1) - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - else - mgmtAddVgroupIntoDb(pDb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + sdbDeleteRow(&oper); } } @@ -230,7 +270,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vgroup_status"); + strcpy(pSchema[cols].name, "vgroup status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -270,13 +310,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vnode_status"); + strcpy(pSchema[cols].name, "vnode status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public_ip"); + strcpy(pSchema[cols].name, "public ip"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -388,109 +428,30 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - pVgroup->vnodeGid[i].vnode = pVgroup->vgId; - } - - return NULL; -} - -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - - if (pDb != NULL) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - } - - // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); - tfree(pVgroup->tableList); - - return NULL; -} - -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - mgmtVgroupActionReset(row, str, size, ssize); - - SVgObj *pVgroup = (SVgObj *) row; - int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb != NULL) { - if (pDb->cfg.maxSessions != oldTables) { - mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); - taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); - } - } - - mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); - - return NULL; -} - -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (size < tsVgUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pVgroup, tsVgUpdateSize); - *ssize = tsVgUpdateSize; - } - - return NULL; -} - -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); - if (pVgroup == NULL) return NULL; - memset(pVgroup, 0, sizeof(SVgObj)); - - int32_t tsVgUpdateSize = pVgroup->updateEnd - (int8_t *) pVgroup; - memcpy(pVgroup, str, tsVgUpdateSize); - - return (void *) pVgroup; -} - -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - memcpy(pVgroup, str, tsVgUpdateSize); - return NULL; -} - -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (pVgroup->idPool) { - taosIdPoolCleanUp(pVgroup->idPool); - pVgroup->idPool = NULL; - } - if (pVgroup->tableList) tfree(pVgroup->tableList); - tfree(row); - return NULL; -} - -void mgmtUpdateVgroup(SVgObj *pVgroup) { - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 0); -} - void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables++; - if (pTable->sid >= 0) + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) { pVgroup->tableList[pTable->sid] = pTable; + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables++; + } + + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); } void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables--; - if (pTable->sid >= 0) + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) { pVgroup->tableList[pTable->sid] = NULL; - taosFreeId(pVgroup->idPool, pTable->sid); + taosFreeId(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables--; + } + + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); } SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + SDbObj *pDb = pVgroup->pDb; if (pDb == NULL) return NULL; SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); @@ -514,6 +475,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SVnodeDesc *vpeerDesc = pVnode->vpeerDesc; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { + vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode); vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].privateIp); } @@ -603,13 +565,23 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { newMsg->contLen = queueMsg->contLen; newMsg->pCont = rpcMallocCont(newMsg->contLen); memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); + queueMsg->pCont = NULL; mgmtAddToShellQueue(newMsg); } else { - sdbDeleteRow(tsVgroupSdb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); } - free(queueMsg); + mgmtFreeQueuedMsg(queueMsg); } static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) { @@ -659,7 +631,15 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; - sdbDeleteRow(tsVgroupSdb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); newMsg->msgType = queueMsg->msgType; @@ -671,50 +651,64 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); mgmtAddToShellQueue(newMsg); - free(queueMsg); + queueMsg->pCont = NULL; + mgmtFreeQueuedMsg(queueMsg); +} + +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + + SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); + if (pVgroup == NULL) { + mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL); } -void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { - void * pNode = NULL; +void mgmtDropAllVgroups(SDbObj *pDropDb) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + int32_t dbNameLen = strlen(pDropDb->name); SVgObj *pVgroup = NULL; + while (1) { pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *vnodeGid = pVgroup->vnodeGid + i; - if (vnodeGid->dnodeId == pDnode->dnodeId) { - mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s", - pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->privateIp), - pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); - vnodeGid->publicIp = pDnode->publicIp; - vnodeGid->privateIp = pDnode->privateIp; - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); - } + if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; } } + + mTrace("db:%s, all vgroups is dropped from sdb", pDropDb->name, numOfTables); } -//static void mgmtProcessVnodeCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *pConn) { -// if (!sdbMaster) { -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// -// SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); -// if (pVgroup == NULL) { -// mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_NOT_ACTIVE_VNODE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateVnodeMsg(pVgroup, pCfg->vnode, &ipSet, NULL); -//} -// \ No newline at end of file +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { + assert(ahandle != NULL); + + if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) { + // TODO: + // mgmtSendAlterVgroupMsg(pVgroup, NULL); + } else { + mgmtAddToShellQueue(ahandle); + } +} \ No newline at end of file diff --git a/src/sdb/CMakeLists.txt b/src/sdb/CMakeLists.txt deleted file mode 100644 index 47ea6e15b8cf5d8ba6f5d24c562e2b133f58f5d0..0000000000000000000000000000000000000000 --- a/src/sdb/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) -INCLUDE_DIRECTORIES(inc) - -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) - AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(sdb ${SRC}) - TARGET_LINK_LIBRARIES(sdb trpc) - IF (TD_CLUSTER) - TARGET_LINK_LIBRARIES(sdb) - ENDIF() -ENDIF () diff --git a/src/sdb/inc/sdbint.h b/src/sdb/inc/sdbint.h deleted file mode 100644 index 30ebe0909c83f5fdefd6105e31f0ecd467d1b77d..0000000000000000000000000000000000000000 --- a/src/sdb/inc/sdbint.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _sdbint_header_ -#define _sdbint_header_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hashint.h" -#include "hashstr.h" -#include "sdb.h" -#include "tchecksum.h" -#include "tlog.h" -#include "trpc.h" -#include "tutil.h" - -#define sdbError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ - } -#define sdbWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbPrint(...) \ - { tprintf("MND-SDB ", 255, __VA_ARGS__); } - -#define mpeerError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-MPEER ", 255, __VA_ARGS__); \ - } -#define mpeerWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerPrint(...) \ - { tprintf("MND-MPEER ", 255, __VA_ARGS__); } - -#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) -#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) -#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) - -#define SDB_MAX_PEERS 4 -#define SDB_DELIMITER 0xFFF00F00 -#define SDB_ENDCOMMIT 0xAFFFAAAF - -typedef struct { - uint64_t swVersion; - int16_t sdbFileVersion; - char reserved[6]; - TSCKSUM checkSum; -} SSdbHeader; - -typedef struct { - char type; - // short rowSize; - char *row; -} SSdbUpdate; - -typedef struct _SSdbTable { - SSdbHeader header; - int maxRows; - int dbId; - int32_t maxRowSize; - char name[TSDB_DB_NAME_LEN]; - char fn[128]; - int keyType; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; - int fd; - void *(*appTool)(char, void *, char *, int, int *); - pthread_mutex_t mutex; - SSdbUpdate * update; - int numOfUpdates; - int updatePos; -} SSdbTable; - -typedef struct { - int64_t id; - int64_t offset; - int rowSize; - void * row; -} SRowMeta; - -typedef struct { - int32_t delimiter; - int32_t rowSize; - int64_t id; - char data[]; -} SRowHead; - -typedef struct { - uint8_t dbId; - char type; - uint64_t version; - short dataLen; - char data[]; -} SForwardMsg; - -extern SSdbTable *tableList[]; -extern int sdbMaxPeers; -extern int sdbNumOfTables; -extern int64_t sdbVersion; - -int sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int dataLen); -int mpeerRetrieveRows(int fd, SSdbTable *pTable, uint64_t version); -void sdbResetTable(SSdbTable *pTable); -extern const int16_t sdbFileVersion; - - -#endif diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c deleted file mode 100644 index fbc41089d1c9917aa5adae26b0052450a44e844d..0000000000000000000000000000000000000000 --- a/src/sdb/src/sdbEngine.c +++ /dev/null @@ -1,1017 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "sdb.h" -#include "sdbint.h" -#include "tutil.h" - -#define abs(x) (((x) < 0) ? -(x) : (x)) - -extern char version[]; -const int16_t sdbFileVersion = 0; -SRpcIpSet *pSdbIpList = NULL; -SRpcIpSet *pSdbPublicIpList = NULL; -SSdbPeer * sdbPeer[SDB_MAX_PEERS]; // first slot for self - -#ifdef CLUSTER -int sdbMaster = 0; -#else -int sdbMaster = 1; -#endif - -void *(*sdbInitIndexFp[])(int maxRows, int dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash}; - -void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash}; - -void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash}; - -void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData}; - -void (*sdbCleanUpIndexFp[])(void *handle) = { - sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash, -}; - -void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = { - sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData, -}; - -SSdbTable *tableList[20]; -int sdbNumOfTables; -int64_t sdbVersion; - -int64_t sdbGetVersion() { - return sdbVersion; -}; - -int32_t sdbGetRunStatus() { - if (!tsIsCluster) { - return SDB_STATUS_SERVING; - } - - if (sdbInited == NULL) { - return SDB_STATUS_OFFLINE; - } - return sdbStatus; -} - -void sdbFinishCommit(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - - off_t offset = lseek(pTable->fd, 0, SEEK_END); - assert(offset == pTable->size); - twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - pTable->size += sizeof(sdbEcommit); -} - -int sdbOpenSdbFile(SSdbTable *pTable) { - struct stat fstat, ofstat; - uint64_t size; - char * dirc = NULL; - char * basec = NULL; - union { - char cversion[64]; - uint64_t iversion; - } swVersion; - - memcpy(swVersion.cversion, version, sizeof(uint64_t)); - - // check sdb.db and .sdb.db status - char fn[128] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - tfree(dirc); - tfree(basec); - if (stat(fn, &ofstat) == 0) { // .sdb.db file exists - if (stat(pTable->fn, &fstat) == 0) { - remove(fn); - } else { - remove(pTable->fn); - rename(fn, pTable->fn); - } - } - - pTable->fd = open(pTable->fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (pTable->fd < 0) { - sdbError("failed to open file:%s", pTable->fn); - return -1; - } - - pTable->size = 0; - stat(pTable->fn, &fstat); - size = sizeof(pTable->header); - - if (fstat.st_size == 0) { - pTable->header.swVersion = swVersion.iversion; - pTable->header.sdbFileVersion = sdbFileVersion; - if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { - sdbError("failed to get file header checksum, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - twrite(pTable->fd, &(pTable->header), size); - pTable->size += size; - sdbFinishCommit(pTable); - } else { - uint32_t sdbEcommit = 0; - off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END); - while (offset > 0) { - read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - if (sdbEcommit == SDB_ENDCOMMIT) { - ftruncate(pTable->fd, offset + sizeof(sdbEcommit)); - break; - } - offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR); - } - lseek(pTable->fd, 0, SEEK_SET); - - ssize_t tsize = read(pTable->fd, &(pTable->header), size); - if (tsize < size) { - sdbError("failed to read sdb file header, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - - if (pTable->header.swVersion != swVersion.iversion) { - sdbWarn("sdb file:%s version not match software version", pTable->fn); - } - - if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { - sdbError("sdb file header is broken since checksum mismatch, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - - pTable->size += size; - // skip end commit symbol - lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); - pTable->size += sizeof(sdbEcommit); - } - - pTable->numOfRows = 0; - - return pTable->fd; -} - -// TODO: Change here -void sdbAddIntoUpdateList(SSdbTable *pTable, char type, char *row) { - pTable->numOfUpdates++; - pTable->updatePos = pTable->numOfUpdates % pTable->maxRows; - - if (pTable->update[pTable->updatePos].type == SDB_TYPE_DELETE) - (*(pTable->appTool))(SDB_TYPE_DESTROY, pTable->update[pTable->updatePos].row, NULL, 0, NULL); - - pTable->update[pTable->updatePos].type = type; - pTable->update[pTable->updatePos].row = row; -} - -int sdbInitTableByFile(SSdbTable *pTable) { - SRowMeta rowMeta; - int numOfDels = 0; - int bytes = 0; - int64_t oldId = 0; - void * pMetaRow = NULL; - int total_size = 0; - int real_size = 0; - int maxAutoIndex = 0; - - oldId = pTable->id; - if (sdbOpenSdbFile(pTable) < 0) return -1; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - - sdbTrace("open sdb file:%s for read", pTable->fn); - - // Loop to read sdb file row by row - while (1) { - memset(rowHead, 0, total_size); - - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); - goto sdb_exit1; - } - - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); - continue; - } - - // sdbTrace("%s id:%ld rowSize:%d", pTable->name, rowHead->id, - // rowHead->rowSize); - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - // TODO: Here may cause pTable->size not end of the file - sdbError("failed to read sdb file:%s id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); - break; - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; - continue; - } - - // Check if the the object exists already - - pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object - if (rowHead->id < 0) { - /* assert(0); */ - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); - } else { - rowMeta.id = rowHead->id; - // TODO: Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t*)rowHead->data); - } - pTable->numOfRows++; - } - } else { // already exists - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); - } - - if (rowHead->id < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); - pTable->numOfRows--; - numOfDels++; - } else { // Reset the object TODO: is it possible to merge reset and - // update ?? - (*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); - } - numOfDels++; - } - - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - } - - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex = maxAutoIndex; - } - - sdbVersion += (pTable->id - oldId); - if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable); - - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - - tfree(rowHead); - return 0; - -sdb_exit1: - tfree(rowHead); - return -1; -} - -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)) { - SSdbTable *pTable = (SSdbTable *)malloc(sizeof(SSdbTable)); - if (pTable == NULL) return NULL; - memset(pTable, 0, sizeof(SSdbTable)); - - int size = sizeof(SSdbUpdate) * maxRows; - pTable->update = (SSdbUpdate *)malloc(size); - if (pTable->update == NULL) { - free(pTable); - return NULL; - }; - memset(pTable->update, 0, size); - - strcpy(pTable->name, name); - pTable->keyType = keyType; - pTable->maxRows = maxRows; - pTable->maxRowSize = maxRowSize; - pTable->appTool = appTool; - sprintf(pTable->fn, "%s/%s.db", directory, pTable->name); - - if (sdbInitIndexFp[keyType] != NULL) pTable->iHandle = (*sdbInitIndexFp[keyType])(maxRows, sizeof(SRowMeta)); - - pthread_mutex_init(&pTable->mutex, NULL); - - if (sdbInitTableByFile(pTable) < 0) return NULL; - - pTable->dbId = sdbNumOfTables++; - tableList[pTable->dbId] = pTable; - - sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d", pTable->name, pTable->numOfRows, sdbNumOfTables); - - return pTable; -} - -SRowMeta *sdbGetRowMeta(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - if (handle == NULL) return NULL; - - pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); - - return pMeta; -} - -void *sdbGetRow(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - if (handle == NULL) return NULL; - - pthread_mutex_lock(&pTable->mutex); - pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); - pthread_mutex_unlock(&pTable->mutex); - - if (pMeta == NULL) return NULL; - - return pMeta->row; -} - -// row here must be encoded string (rowSize > 0) or the object it self (rowSize -// = 0) -int64_t sdbInsertRow(void *handle, void *row, int rowSize) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta rowMeta; - int64_t id = -1; - void * pObj = NULL; - int total_size = 0; - int real_size = 0; - /* char action = SDB_TYPE_INSERT; */ - - if (pTable == NULL) { - sdbError("sdb tables is null"); - return -1; - } - - if ((pTable->keyType != SDB_KEYTYPE_AUTO) || *((int64_t *)row)) - if (sdbGetRow(handle, row)) { - if (strcmp(pTable->name, "mnode") == 0) { - /* - * The first mnode created when the system just start, so the insert action may failed - * see sdbPeer.c : sdbInitPeers - */ - pTable->id++; - sdbVersion++; - sdbPrint("table:%s, record:%s already exist, think it successed, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - return 0; - } else { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); - break; - } - return -1; - } - } - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb: %s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - if (rowSize == 0) { // object is created already - pObj = row; - } else { // encoded string, to create object - pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, rowSize, NULL); - } - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - // TODO:here need to change - *((uint32_t *)pObj) = ++pTable->autoIndex; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - // update in SDB layer - rowMeta.id = pTable->id; - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - - /* Update the disk content */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - pTable->size += real_size; - sdbFinishCommit(pTable); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_INSERT, rowMeta.row); - - pTable->numOfRows++; - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - default: - sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - } - - id = rowMeta.id; - } else { - sdbError("table:%s, failed to insert record", pTable->name); - } - - tfree(rowHead); - - pthread_mutex_unlock(&pTable->mutex); - - /* callback function to update the MGMT layer */ - if (id >= 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); - - return id; -} - -// row here can be object or null-terminated string -int sdbDeleteRow(void *handle, void *row) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int code = -1; - void * pMetaRow = NULL; - SRowHead * rowHead = NULL; - int rowSize = 0; - int total_size = 0; - /* char action = SDB_TYPE_DELETE; */ - - if (pTable == NULL) return -1; - - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, delete failed", pTable->name); - return -1; - } - - pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - rowSize = strlen((char *)row) + 1; - break; - case SDB_KEYTYPE_UINT32: - rowSize = sizeof(uint32_t); - break; - case SDB_KEYTYPE_AUTO: - rowSize = sizeof(uint64_t); - break; - default: - return -1; - } - - total_size = sizeof(SRowHead) + rowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { - pTable->id++; - sdbVersion++; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); - - pTable->numOfRows--; - // TODO:Change the update list here - sdbAddIntoUpdateList(pTable, SDB_TYPE_DELETE, pMetaRow); - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, sdbVersion, pTable->id, pTable->numOfRows); - break; - } - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); - - code = 0; - } - - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - // callback function of the delete - if (code == 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); - - return code; -} - -// row here can be the object or the string info (encoded string) -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int code = -1; - int total_size = 0; - int real_size = 0; - /* char action = SDB_TYPE_UPDATE; */ - - if (pTable == NULL || row == NULL) return -1; - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, (char *) row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *) row), sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to update record:%d, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, *(int32_t *) row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to update record, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, sdbVersion, pTable->id); - break; - } - return -1; - } - - void *pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - if (!isUpdated) { - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer - } - - if (pMetaRow != row) { - memcpy(rowHead->data, row, updateSize); - rowHead->rowSize = updateSize; - } else { - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMetaRow, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - ; - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - - // write to the new position - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, pMetaRow); - code = 0; - } - - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - return code; -} - -// row here must be the instruction string -int sdbBatchUpdateRow(void *handle, void *row, int rowSize) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int total_size = 0; - /* char action = SDB_TYPE_BATCH_UPDATE; */ - - if (pTable == NULL || row == NULL || rowSize <= 0) return -1; - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, batch update failed", pTable->name); - return -1; - } - - void *pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - - pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_BATCH_UPDATE, row, rowSize) == 0) { - /* // write action */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - - (*(pTable->appTool))(SDB_TYPE_BEFORE_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - - void *next_row = pMetaRow; - while (next_row != NULL) { - pTable->id++; - sdbVersion++; - - void *last_row = next_row; - next_row = (*(pTable->appTool))(SDB_TYPE_BATCH_UPDATE, last_row, (char *)row, rowSize, 0); - memset(rowHead, 0, sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM)); - - // update in current layer - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - - // write to disk - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, last_row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - taosCalcChecksumAppend(0, (uint8_t *)rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pMeta->rowSize = rowHead->rowSize; - lseek(pTable->fd, pTable->size, SEEK_SET); - twrite(pTable->fd, rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pTable->size += (sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, last_row); - - if (next_row != NULL) { - pMeta = sdbGetRowMeta(handle, next_row); - } - } - - sdbFinishCommit(pTable); - - (*(pTable->appTool))(SDB_TYPE_AFTER_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - } - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - return 0; -} - -void sdbCloseTable(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - void * pNode = NULL; - void * row; - - if (pTable == NULL) return; - - while (1) { - pNode = sdbFetchRow(handle, pNode, &row); - if (row == NULL) break; - (*(pTable->appTool))(SDB_TYPE_DESTROY, row, NULL, 0, NULL); - } - - if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); - - if (pTable->fd) tclose(pTable->fd); - - pthread_mutex_destroy(&pTable->mutex); - - sdbNumOfTables--; - sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->name, pTable->id, sdbNumOfTables); - - tfree(pTable->update); - tfree(pTable); -} - -void sdbResetTable(SSdbTable *pTable) { - /* SRowHead rowHead; */ - SRowMeta rowMeta; - int bytes; - int total_size = 0; - int real_size = 0; - SRowHead *rowHead = NULL; - void * pMetaRow = NULL; - int64_t oldId = pTable->id; - int oldNumOfRows = pTable->numOfRows; - - if (sdbOpenSdbFile(pTable) < 0) return; - pTable->numOfRows = oldNumOfRows; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->name); - return; - } - - sdbPrint("open sdb file:%s for reset table", pTable->fn); - - while (1) { - memset(rowHead, 0, total_size); - - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); - tfree(rowHead); - return; - } - - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s for reset, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); - continue; - } - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - sdbError("failed to read sdb file:%s for reset, id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); - break; - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; - continue; - } - - if (abs(rowHead->id) > oldId) { // not operated - pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object - if (rowHead->id < 0) { - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); - } else { - rowMeta.id = rowHead->id; - // TODO:Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - pTable->numOfRows++; - - (*pTable->appTool)(SDB_TYPE_INSERT, rowMeta.row, NULL, 0, NULL); - } - } else { // already exists - if (rowHead->id < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); - pTable->numOfRows--; - } else { // update the object - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, rowHead->data, rowHead->rowSize, NULL); - } - } - } - - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - } - - sdbVersion += (pTable->id - oldId); - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - - tfree(rowHead); - - sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); -} - -// TODO:A problem here :use snapshot file to sync another node will cause -// problem -void sdbSaveSnapShot(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - void * pNode = NULL; - int total_size = 0; - int real_size = 0; - int size = 0; - int numOfRows = 0; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - char * dirc = NULL; - char * basec = NULL; - /* char action = SDB_TYPE_INSERT; */ - - if (pTable == NULL) return; - - sdbTrace("Table:%s, save the snapshop", pTable->name); - - char fn[128] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - int fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - tfree(dirc); - tfree(basec); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->name); - return; - } - memset(rowHead, 0, size); - - // Write the header - twrite(fd, &(pTable->header), sizeof(SSdbHeader)); - size += sizeof(SSdbHeader); - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - - while (1) { - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMeta->row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while save sdb %s snapshot", pTable->name); - tfree(rowHead); - return; - } - - /* write(fd, &action, sizeof(action)); */ - /* size += sizeof(action); */ - twrite(fd, rowHead, real_size); - size += real_size; - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - numOfRows++; - } - - tfree(rowHead); - - // Remove the old file - tclose(pTable->fd); - remove(pTable->fn); - // Rename the .sdb.db file to sdb.db file - rename(fn, pTable->fn); - pTable->fd = fd; - pTable->size = size; - pTable->numOfRows = numOfRows; - - fdatasync(pTable->fd); -} - -void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - *ppRow = NULL; - if (pTable == NULL) return NULL; - - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) return NULL; - - *ppRow = pMeta->row; - - return pNode; -} - -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } - -int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } diff --git a/src/sdb/src/sdbstr.c b/src/sdb/src/sdbstr.c deleted file mode 100644 index 59c01eb15a95a7354b5f37acb1d521a5e89c100d..0000000000000000000000000000000000000000 --- a/src/sdb/src/sdbstr.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#define _DEFAULT_SOURCE -#include "sdbint.h" - -int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; -void (*mpeerCleanUpMnodesFp)() = NULL; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; - -char *sdbStatusStr[] = { - "offline", - "unsynced", - "syncing", - "serving", - "null" -}; - -char *sdbRoleStr[] = { - "unauthed", - "undecided", - "master", - "slave", - "null" -}; - -int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); - } else { - return 0; - } -} - -int32_t sdbInitPeers(char *directory) { - if (mpeerInitMnodesFp) { - return (*mpeerInitMnodesFp)(directory); - } else { - return 0; - } -} - -void sdbCleanUpPeers() { - if (mpeerCleanUpMnodesFp) { - (*mpeerCleanUpMnodesFp)(); - } -} diff --git a/src/sdb/inc/hashint.h b/src/util/inc/hashint.h similarity index 100% rename from src/sdb/inc/hashint.h rename to src/util/inc/hashint.h diff --git a/src/sdb/inc/hashstr.h b/src/util/inc/hashstr.h similarity index 100% rename from src/sdb/inc/hashstr.h rename to src/util/inc/hashstr.h diff --git a/src/util/inc/tglobalcfg.h b/src/util/inc/tglobalcfg.h index 993992ffcbf9a33b00761dd20141224f912db958..664eb163734eef0eccdb65a315ee89a816b9f530 100644 --- a/src/util/inc/tglobalcfg.h +++ b/src/util/inc/tglobalcfg.h @@ -103,13 +103,7 @@ extern int tsReplications; extern int tsNumOfMPeers; extern int tsMaxShellConns; -extern int tsMaxAccounts; -extern int tsMaxUsers; -extern int tsMaxDbs; extern int tsMaxTables; -extern int tsMaxDnodes; -extern int tsMaxVGroups; -extern char tsMgmtZone[]; extern char tsLocalIp[]; extern char tsDefaultDB[]; diff --git a/src/util/inc/tidpool.h b/src/util/inc/tidpool.h index 08f9d364671a5d126e699c26cb92e74365be0c78..bf352516310a1356f37359ab3052814d726ab8f0 100644 --- a/src/util/inc/tidpool.h +++ b/src/util/inc/tidpool.h @@ -34,11 +34,7 @@ void taosIdPoolCleanUp(void *handle); int taosIdPoolNumOfUsed(void *handle); -void taosIdPoolReinit(void *handle); - -void taosIdPoolMarkStatus(void *handle, int id, int status); - -void taosIdPoolSetFreeList(void *handle); +void taosIdPoolMarkStatus(void *handle, int id); #ifdef __cplusplus } diff --git a/src/util/inc/tlog.h b/src/util/inc/tlog.h index 350a2c700e0dddb15f1726eb50d47117f9228603..3a327b0a471c4671c684d80e0215f0c863239418 100644 --- a/src/util/inc/tlog.h +++ b/src/util/inc/tlog.h @@ -239,6 +239,25 @@ extern uint32_t cdebugFlag; #define monitorLWarn(...) taosLogWarn(__VA_ARGS__) monitorWarn(__VA_ARGS__) #define monitorLPrint(...) taosLogPrint(__VA_ARGS__) monitorPrint(__VA_ARGS__) +#define sdbError(...) \ + if (sdbDebugFlag & DEBUG_ERROR) { \ + tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ + } +#define sdbWarn(...) \ + if (sdbDebugFlag & DEBUG_WARN) { \ + tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbTrace(...) \ + if (sdbDebugFlag & DEBUG_TRACE) { \ + tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbPrint(...) \ + { tprintf("MND-SDB ", 255, __VA_ARGS__); } + +#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) +#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) +#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) + #ifdef __cplusplus } #endif diff --git a/src/sdb/src/hashint.c b/src/util/src/hashint.c similarity index 100% rename from src/sdb/src/hashint.c rename to src/util/src/hashint.c diff --git a/src/sdb/src/hashstr.c b/src/util/src/hashstr.c similarity index 100% rename from src/sdb/src/hashstr.c rename to src/util/src/hashstr.c diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index bbea30207cfbb275caf37673afd925bea63559d7..02de88881d9dcfdc2b0ac45114bd59036faa4d26 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -112,13 +112,7 @@ int tsReplications = TSDB_REPLICA_MIN_NUM; int tsNumOfMPeers = 3; int tsMaxShellConns = 2000; -int tsMaxAccounts = 100; -int tsMaxUsers = 1000; -int tsMaxDbs = 1000; -int tsMaxTables = 650000; -int tsMaxDnodes = 1000; -int tsMaxVGroups = 1000; -char tsMgmtZone[16] = "rzone"; +int tsMaxTables = 100000; char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsDefaultDB[TSDB_DB_NAME_LEN] = {0}; @@ -612,28 +606,10 @@ static void doInitGlobalConfig() { 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); // mgmt configs - tsInitConfigOption(cfg++, "mgmtZone", tsMgmtZone, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 0, 0, 16, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxAccounts", &tsMaxAccounts, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxUsers", &tsMaxUsers, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDbs", &tsMaxDbs, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 10000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxTables", &tsMaxTables, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 100000000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDnodes", &tsMaxDnodes, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxVGroups", &tsMaxVGroups, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000000, 0, TSDB_CFG_UTYPE_NONE); - + tsInitConfigOption(cfg++, "minSlidingTime", &tsMinSlidingTime, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000, 0, TSDB_CFG_UTYPE_MS); diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index c50c38aa3cabd74266d4164f0ebdf4581ce670f6..5ed6b8cf232eb1fb774af75a9738f6a403cda93c 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -15,88 +15,81 @@ #include "os.h" #include "tlog.h" +#include typedef struct { int maxId; int numOfFree; int freeSlot; - int * freeList; + bool * freeList; pthread_mutex_t mutex; } id_pool_t; void *taosInitIdPool(int maxId) { - id_pool_t *pIdPool; - int * idList, i; - - if (maxId < 3) maxId = 3; - - pIdPool = (id_pool_t *)malloc(sizeof(id_pool_t)); + id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); - if (idList == NULL) { + pIdPool->freeList = calloc(maxId, sizeof(bool)); + if (pIdPool->freeList == NULL) { free(pIdPool); return NULL; } - memset(pIdPool, 0, sizeof(id_pool_t)); pIdPool->maxId = maxId; - pIdPool->numOfFree = maxId - 1; + pIdPool->numOfFree = maxId; pIdPool->freeSlot = 0; - pIdPool->freeList = idList; pthread_mutex_init(&pIdPool->mutex, NULL); - for (i = 1; i < maxId; ++i) idList[i - 1] = i; - pTrace("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); - return (void *)pIdPool; + return pIdPool; } int taosAllocateId(void *handle) { - id_pool_t *pIdPool; - int id = -1; - if (handle == NULL) return id; - - pIdPool = (id_pool_t *)handle; - - if (pIdPool->maxId < 3) pError("pool:%p is messed up, maxId:%d", pIdPool, pIdPool->maxId); + id_pool_t *pIdPool = handle; + if (handle == NULL) { + return -1; + } - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + int slot = -1; + pthread_mutex_lock(&pIdPool->mutex); if (pIdPool->numOfFree > 0) { - id = pIdPool->freeList[pIdPool->freeSlot]; - pIdPool->freeSlot = (pIdPool->freeSlot + 1) % pIdPool->maxId; - pIdPool->numOfFree--; + for (int i = 0; i < pIdPool->maxId; ++i) { + slot = (i + pIdPool->freeSlot) % pIdPool->maxId; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; + pIdPool->freeSlot = slot + 1; + pIdPool->numOfFree--; + break; + } + } } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); - - return id; + pthread_mutex_unlock(&pIdPool->mutex); + return slot + 1; } void taosFreeId(void *handle, int id) { - id_pool_t *pIdPool; - int slot; + id_pool_t *pIdPool = handle; + if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; - if (pIdPool->freeList == NULL || pIdPool->maxId == 0) return; - if (id <= 0 || id >= pIdPool->maxId) return; - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + pthread_mutex_lock(&pIdPool->mutex); - slot = (pIdPool->freeSlot + pIdPool->numOfFree) % pIdPool->maxId; - pIdPool->freeList[slot] = id; - pIdPool->numOfFree++; + int slot = (id - 1) % pIdPool->maxId; + if (pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = false; + pIdPool->numOfFree++; + } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); } void taosIdPoolCleanUp(void *handle) { - id_pool_t *pIdPool; + id_pool_t *pIdPool = handle; - if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; + if (pIdPool == NULL) return; pTrace("pool:%p is cleaned", pIdPool); @@ -110,42 +103,21 @@ void taosIdPoolCleanUp(void *handle) { } int taosIdPoolNumOfUsed(void *handle) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - return pIdPool->maxId - pIdPool->numOfFree - 1; -} - -void taosIdPoolReinit(void *handle) { - id_pool_t *pIdPool; - - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 0; i < pIdPool->maxId; ++i) pIdPool->freeList[i] = 0; + id_pool_t *pIdPool = handle; + return pIdPool->maxId - pIdPool->numOfFree; } -void taosIdPoolMarkStatus(void *handle, int id, int status) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - pIdPool->freeList[id] = status; -} +void taosIdPoolMarkStatus(void *handle, int id) { + id_pool_t *pIdPool = handle; + pthread_mutex_lock(&pIdPool->mutex); -void taosIdPoolSetFreeList(void *handle) { - id_pool_t *pIdPool; - int pos = 0; - - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 1; i < pIdPool->maxId; ++i) { - if (pIdPool->freeList[i] == 0) { - pIdPool->freeList[pos] = i; - pIdPool->numOfFree++; - pos++; - } + int slot = (id - 1) % pIdPool->maxId; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; + pIdPool->numOfFree--; } + + pthread_mutex_unlock(&pIdPool->mutex); } int taosUpdateIdPool(id_pool_t *handle, int maxId) { @@ -154,18 +126,14 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return -1; } - int *idList, i; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); + int *idList = calloc(maxId, sizeof(bool)); if (idList == NULL) { return -1; } - for (i = 1; i < maxId; ++i) { - idList[i - 1] = i; - } - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + pthread_mutex_lock(&pIdPool->mutex); - memcpy(idList, pIdPool->freeList, sizeof(int) * (size_t)pIdPool->maxId); + memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); pIdPool->maxId = maxId; @@ -173,7 +141,7 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { pIdPool->freeList = idList; free(oldIdList); - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); return 0; }