Skip to content

TDengine
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
TDengine
TDengine
提交 d3861996 编写于 作者: H Hongze Cheng
浏览文件
操作

Merge branch '3.0' into feature/vnode

上级 d86a5ca3 278fc0f6
展开全部 显示空白变更内容
内联 并排
Showing with 2670 addition and 461 deletion
include/common/taosmsg.h
浏览文件 @ d3861996
......@@ -81,6 +81,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE, "drop-stable" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STABLE_VGROUP, "stable-vgroup" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" )
......@@ -153,7 +154,8 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY9, "dummy9" )
#define TSDB_IE_TYPE_DNODE_EXT 6
#define TSDB_IE_TYPE_DNODE_STATE 7
enum _mgmt_table {
typedef enum _mgmt_table {
TSDB_MGMT_TABLE_START,
TSDB_MGMT_TABLE_ACCT,
TSDB_MGMT_TABLE_USER,
TSDB_MGMT_TABLE_DB,
......@@ -175,7 +177,7 @@ enum _mgmt_table {
TSDB_MGMT_TABLE_TP,
TSDB_MGMT_TABLE_FUNCTION,
TSDB_MGMT_TABLE_MAX,
};
} EShowType;
#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1
#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2
......@@ -352,11 +354,9 @@ typedef struct {
} SUpdateTableTagValMsg;
typedef struct {
char clientVersion[TSDB_VERSION_LEN];
char msgVersion[TSDB_VERSION_LEN];
char db[TSDB_TABLE_FNAME_LEN];
char appName[TSDB_APPNAME_LEN];
int32_t pid;
char app[TSDB_APP_NAME_LEN];
char db[TSDB_DB_NAME_LEN];
} SConnectMsg;
typedef struct SEpSet {
......@@ -367,14 +367,13 @@ typedef struct SEpSet {
} SEpSet;
typedef struct {
char acctId[TSDB_ACCT_ID_LEN];
char serverVersion[TSDB_VERSION_LEN];
char clusterId[TSDB_CLUSTER_ID_LEN];
int8_t writeAuth;
int8_t superAuth;
int8_t reserved1;
int8_t reserved2;
int32_t acctId;
int32_t clusterId;
int32_t connId;
int8_t superAuth;
int8_t readAuth;
int8_t writeAuth;
int8_t reserved[5];
SEpSet epSet;
} SConnectRsp;
......@@ -874,22 +873,22 @@ typedef struct {
} SStreamDesc;
typedef struct {
char clientVer[TSDB_VERSION_LEN];
uint32_t connId;
int32_t connId;
int32_t pid;
int32_t numOfQueries;
int32_t numOfStreams;
char appName[TSDB_APPNAME_LEN];
char app[TSDB_APP_NAME_LEN];
char pData[];
} SHeartBeatMsg;
typedef struct {
uint32_t queryId;
uint32_t streamId;
uint32_t totalDnodes;
uint32_t onlineDnodes;
uint32_t connId;
int32_t connId;
int32_t queryId;
int32_t streamId;
int32_t totalDnodes;
int32_t onlineDnodes;
int8_t killConnection;
int8_t reserved[3];
SEpSet epSet;
} SHeartBeatRsp;
......
include/dnode/mnode/mnode.h
浏览文件 @ d3861996
......@@ -57,6 +57,7 @@ typedef struct {
int32_t sver;
int32_t statusInterval;
int32_t mnodeEqualVnodeNum;
int32_t shellActivityTimer;
char *timezone;
char *locale;
char *charset;
......
include/libs/wal/wal.h
浏览文件 @ d3861996
......@@ -39,6 +39,14 @@ typedef enum {
} EWalType;
typedef struct {
//union {
//uint32_t info;
//struct {
//uint32_t sver:3;
//uint32_t msgtype: 5;
//uint32_t reserved : 24;
//};
//};
int8_t sver;
uint8_t msgType;
int8_t reserved[2];
......@@ -71,13 +79,17 @@ typedef struct {
#define WAL_FILESET_MAX 128
#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2)
#define WAL_CUR_POS_READ_ONLY 1
#define WAL_CUR_FILE_READ_ONLY 2
#define WAL_CUR_POS_WRITABLE 1
#define WAL_CUR_FILE_WRITABLE 2
#define WAL_CUR_FAILED 4
typedef struct SWal {
// cfg
int32_t vgId;
int32_t fsyncPeriod; // millisecond
int32_t fsyncSeq;
int32_t rollPeriod; // second
int64_t segSize;
EWalType level;
//reference
int64_t refId;
......@@ -86,7 +98,7 @@ typedef struct SWal {
int64_t curIdxTfd;
//current version
int64_t curVersion;
int64_t curOffset;
int64_t curLogOffset;
//current file version
int64_t curFileFirstVersion;
int64_t curFileLastVersion;
......@@ -94,8 +106,10 @@ typedef struct SWal {
int64_t firstVersion;
int64_t snapshotVersion;
int64_t lastVersion;
//fsync status
int32_t fsyncSeq;
int64_t lastFileName;
//roll status
int64_t lastRollSeq;
int64_t lastFileWriteSize;
//ctl
int32_t curStatus;
pthread_mutex_t mutex;
......@@ -119,12 +133,10 @@ int32_t walAlter(SWal *, SWalCfg *pCfg);
void walClose(SWal *);
// write
//int64_t walWriteWithMsgType(SWal*, int8_t msgType, void* body, int32_t bodyLen);
int64_t walWrite(SWal *, int64_t index, uint8_t msgType, void *body, int32_t bodyLen);
//int64_t walWriteBatch(SWal *, void **bodies, int32_t *bodyLen, int32_t batchSize);
void walFsync(SWal *, bool force);
// apis for lifecycle management
void walFsync(SWal *, bool force);
int32_t walCommit(SWal *, int64_t ver);
// truncate after
int32_t walRollback(SWal *, int64_t ver);
......
include/util/tdef.h
浏览文件 @ d3861996
......@@ -174,7 +174,7 @@ do { \
#define TSDB_MAX_SQL_SHOW_LEN 512
#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb
#define TSDB_APPNAME_LEN TSDB_UNI_LEN
#define TSDB_APP_NAME_LEN TSDB_UNI_LEN
/**
* In some scenarios uint16_t (0~65535) is used to store the row len.
......
include/util/tfile.h
浏览文件 @ d3861996
......@@ -28,6 +28,7 @@ void tfCleanup();
// the same syntax as UNIX standard open/close/read/write
// but FD is int64_t and will never be reused
int64_t tfOpenRead(const char *pathname);
int64_t tfOpenReadWrite(const char *pathname);
int64_t tfOpenCreateWrite(const char *pathname);
int64_t tfOpenCreateWriteAppend(const char *pathname);
......
include/util/tutil.h
浏览文件 @ d3861996
......@@ -41,8 +41,10 @@ char * paGetToken(char *src, char **token, int32_t *tokenLen);
int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]);
int32_t taosHexStrToByteArray(char hexstr[], char bytes[]);
char * taosIpStr(uint32_t ipInt);
char *taosIpStr(uint32_t ipInt);
uint32_t ip2uint(const char *const ip_addr);
void taosIp2String(uint32_t ip, char *str);
void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
MD5_CTX context;
......
source/dnode/mgmt/impl/CMakeLists.txt
浏览文件 @ d3861996
......@@ -14,3 +14,7 @@ target_include_directories(
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})
\ No newline at end of file
source/dnode/mgmt/impl/src/dndMnode.c
浏览文件 @ d3861996
......@@ -334,6 +334,7 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) {
pOption->sver = pDnode->opt.sver;
pOption->statusInterval = pDnode->opt.statusInterval;
pOption->mnodeEqualVnodeNum = pDnode->opt.mnodeEqualVnodeNum;
pOption->shellActivityTimer = pDnode->opt.shellActivityTimer;
pOption->timezone = pDnode->opt.timezone;
pOption->charset = pDnode->opt.charset;
pOption->locale = pDnode->opt.locale;
......@@ -675,7 +676,7 @@ void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) {
if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
rpcSendResponse(&rsp);
rpcFreeCont(pMsg->pCont);
......
source/dnode/mgmt/impl/src/dndTransport.c
浏览文件 @ d3861996
......@@ -76,6 +76,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg;
......@@ -157,7 +158,7 @@ static int32_t dndInitClient(SDnode *pDnode) {
rpcInit.label = "DND-C";
rpcInit.numOfThreads = 1;
rpcInit.cfp = dndProcessResponse;
rpcInit.sessions = 8;
rpcInit.sessions = 1024;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000;
rpcInit.user = INTERNAL_USER;
......
source/dnode/mgmt/impl/test/CMakeLists.txt 0 → 100644
浏览文件 @ d3861996
add_subdirectory(test01)
\ No newline at end of file
source/dnode/mgmt/impl/test/test01/CMakeLists.txt 0 → 100644
浏览文件 @ d3861996
add_executable(dndTest01 "")
target_sources(dndTest01
PRIVATE
"dndTest01.cpp"
"../util/dndTestDeploy.cpp"
)
target_link_libraries(
dndTest01
PUBLIC dnode
PUBLIC util
PUBLIC os
PUBLIC gtest_main
)
target_include_directories(dndTest01
PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt"
"${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
"${CMAKE_CURRENT_SOURCE_DIR}/../util"
)
enable_testing()
add_test(
NAME dndTest01
COMMAND dndTest01
)
source/dnode/mgmt/impl/test/test01/dndTest01.cpp 0 → 100644
浏览文件 @ d3861996
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "dndTestDeploy.h"
class DndTest01 : public ::testing::Test {
protected:
void SetUp() override {
pServer = createServer("/tmp/dndTest01");
pClient = createClient("root", "taosdata");
}
void TearDown() override {
dropServer(pServer);
dropClient(pClient);
}
SServer* pServer;
SClient* pClient;
};
TEST_F(DndTest01, connectMsg) {
SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg));
pReq->pid = htonl(1234);
strcpy(pReq->app, "test01");
strcpy(pReq->db, "");
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SConnectMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT;
sendMsg(pClient, &rpcMsg);
SConnectRsp* pRsp = (SConnectRsp*)pClient->pRsp;
ASSERT(pRsp);
pRsp->acctId = htonl(pRsp->acctId);
pRsp->clusterId = htonl(pRsp->clusterId);
pRsp->connId = htonl(pRsp->connId);
pRsp->epSet.port[0] = htonl(pRsp->epSet.port[0]);
EXPECT_EQ(pRsp->acctId, 1);
EXPECT_GT(pRsp->clusterId, 0);
EXPECT_GT(pRsp->connId, 1);
EXPECT_EQ(pRsp->superAuth, 1);
EXPECT_EQ(pRsp->readAuth, 1);
EXPECT_EQ(pRsp->writeAuth, 1);
EXPECT_EQ(pRsp->epSet.inUse, 0);
EXPECT_EQ(pRsp->epSet.numOfEps, 1);
EXPECT_EQ(pRsp->epSet.port[0], 9527);
EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost");
}
// TEST_F(DndTest01, heartbeatMsg) {
// SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg));
// pReq->connId = htonl(1);
// pReq->pid = htonl(1234);
// pReq->numOfQueries = htonl(0);
// pReq->numOfStreams = htonl(0);
// strcpy(pReq->app, "test01");
// SRpcMsg rpcMsg = {0};
// rpcMsg.pCont = pReq;
// rpcMsg.contLen = sizeof(SHeartBeatMsg);
// rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT;
// sendMsg(pClient, &rpcMsg);
// SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pClient->pRsp;
// ASSERT(pRsp);
// pRsp->epSet.port[0] = htonl(pRsp->epSet.port[0]);
// EXPECT_EQ(htonl(pRsp->connId), 1);
// EXPECT_GT(htonl(pRsp->queryId), 0);
// EXPECT_GT(htonl(pRsp->streamId), 1);
// EXPECT_EQ(htonl(pRsp->totalDnodes), 1);
// EXPECT_EQ(htonl(pRsp->onlineDnodes), 1);
// EXPECT_EQ(pRsp->killConnection, 0);
// EXPECT_EQ(pRsp->epSet.inUse, 0);
// EXPECT_EQ(pRsp->epSet.numOfEps, 1);
// EXPECT_EQ(pRsp->epSet.port[0], 9527);
// EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost");
// }
source/dnode/mgmt/impl/test/util/dndTestDeploy.cpp 0 → 100644
浏览文件 @ d3861996
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "dndTestDeploy.h"
void initLog(char *path) {
mDebugFlag = 207;
char temp[PATH_MAX];
snprintf(temp, PATH_MAX, "%s/taosdlog", path);
if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
printf("failed to init log file\n");
}
}
void* runServer(void* param) {
SServer* pServer = (SServer*)param;
while (1) {
taosMsleep(100);
pthread_testcancel();
}
}
void initOption(SDnodeOpt* pOption, char *path) {
pOption->sver = 1;
pOption->numOfCores = 1;
pOption->numOfSupportMnodes = 1;
pOption->numOfSupportVnodes = 1;
pOption->numOfSupportQnodes = 1;
pOption->statusInterval = 1;
pOption->mnodeEqualVnodeNum = 1;
pOption->numOfThreadsPerCore = 1;
pOption->ratioOfQueryCores = 1;
pOption->maxShellConns = 1000;
pOption->shellActivityTimer = 30;
pOption->serverPort = 9527;
strcpy(pOption->dataDir, path);
strcpy(pOption->localEp, "localhost:9527");
strcpy(pOption->localFqdn, "localhost");
strcpy(pOption->firstEp, "localhost:9527");
taosRemoveDir(path);
taosMkDir(path);
}
SServer* createServer(char *path) {
SDnodeOpt option = {0};
initOption(&option, path);
SDnode* pDnode = dndInit(&option);
ASSERT(pDnode);
SServer* pServer = (SServer*)calloc(1, sizeof(SServer));
ASSERT(pServer);
pServer->pDnode = pDnode;
pServer->threadId = taosCreateThread(runServer, pServer);
ASSERT(pServer->threadId);
return pServer;
}
void dropServer(SServer* pServer) {
if (pServer->threadId != NULL) {
taosDestoryThread(pServer->threadId);
}
}
void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
SClient* pClient = (SClient*)parent;
pClient->pRsp = pMsg;
taosMsleep(100000);
tsem_post(&pClient->sem);
}
SClient* createClient(char *user, char *pass) {
SClient* pClient = (SClient*)calloc(1, sizeof(SClient));
ASSERT(pClient);
char secretEncrypt[32] = {0};
taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt);
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.label = "DND-C";
rpcInit.numOfThreads = 1;
rpcInit.cfp = processClientRsp;
rpcInit.sessions = 1024;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = 30 * 1000;
rpcInit.user = user;
rpcInit.ckey = "key";
rpcInit.parent = pClient;
rpcInit.secret = (char*)secretEncrypt;
rpcInit.parent = pClient;
// rpcInit.spi = 1;
pClient->clientRpc = rpcOpen(&rpcInit);
ASSERT(pClient->clientRpc);
tsem_init(&pClient->sem, 0, 0);
return pClient;
}
void dropClient(SClient* pClient) {
tsem_destroy(&pClient->sem);
rpcClose(pClient->clientRpc);
}
void sendMsg(SClient* pClient, SRpcMsg* pMsg) {
SEpSet epSet = {0};
epSet.inUse = 0;
epSet.numOfEps = 1;
epSet.port[0] = 9527;
strcpy(epSet.fqdn[0], "localhost");
rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL);
tsem_wait(&pClient->sem);
}
source/dnode/mgmt/impl/test/util/dndTestDeploy.h 0 → 100644
浏览文件 @ d3861996
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include "os.h"
#include "dnode.h"
#include "taosmsg.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tnote.h"
#include "trpc.h"
#include "tthread.h"
#include "ulog.h"
typedef struct {
SDnode* pDnode;
pthread_t* threadId;
} SServer;
typedef struct {
void* clientRpc;
SRpcMsg* pRsp;
tsem_t sem;
} SClient;
SServer* createServer(char* path);
void dropServer(SServer* pServer);
SClient* createClient(char *user, char *pass);
void dropClient(SClient* pClient);
void sendMsg(SClient* pClient, SRpcMsg* pMsg);
source/dnode/mnode/impl/inc/mndDb.h
浏览文件 @ d3861996
......@@ -24,6 +24,8 @@ extern "C" {
int32_t mndInitDb(SMnode *pMnode);
void mndCleanupDb(SMnode *pMnode);
SDbObj *mndAcquireDb(SMnode *pMnode, char *db);
void mndReleaseDb(SMnode *pMnode, SDbObj *pDb);
#ifdef __cplusplus
}
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ d3861996
......@@ -185,9 +185,11 @@ typedef struct SUserObj {
char acct[TSDB_USER_LEN];
int64_t createdTime;
int64_t updateTime;
int8_t rootAuth;
int8_t superAuth;
int8_t readAuth;
int8_t writeAuth;
int32_t acctId;
SHashObj *prohibitDbHash;
SAcctObj *pAcct;
} SUserObj;
typedef struct {
......@@ -281,18 +283,20 @@ typedef struct SFuncObj {
int16_t type;
} SFuncObj;
typedef struct {
typedef struct SShowObj SShowObj;
typedef struct SShowObj {
int8_t type;
int8_t maxReplica;
int16_t numOfColumns;
int32_t index;
int32_t id;
int32_t rowSize;
int32_t numOfRows;
int32_t numOfReads;
uint16_t payloadLen;
void *pIter;
void *pVgIter;
void **ppShow;
SMnode *pMnode;
SShowObj **ppShow;
char db[TSDB_FULL_DB_NAME_LEN];
int16_t offset[TSDB_MAX_COLUMNS];
int32_t bytes[TSDB_MAX_COLUMNS];
......@@ -301,6 +305,8 @@ typedef struct {
typedef struct SMnodeMsg {
char user[TSDB_USER_LEN];
char db[TSDB_FULL_DB_NAME_LEN];
int32_t acctId;
SMnode *pMnode;
int16_t received;
int16_t successed;
......
source/dnode/mnode/impl/inc/mndInt.h
浏览文件 @ d3861996
......@@ -18,15 +18,19 @@
#include "mndDef.h"
#include "sdb.h"
#include "tcache.h"
#include "tqueue.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef int32_t (*MndMsgFp)(SMnode *pMnode, SMnodeMsg *pMsg);
typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg);
typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode);
typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta);
typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter);
typedef struct {
const char *name;
......@@ -34,6 +38,19 @@ typedef struct {
MndCleanupFp cleanupFp;
} SMnodeStep;
typedef struct {
int32_t showId;
ShowMetaFp metaFps[TSDB_MGMT_TABLE_MAX];
ShowRetrieveFp retrieveFps[TSDB_MGMT_TABLE_MAX];
ShowFreeIterFp freeIterFps[TSDB_MGMT_TABLE_MAX];
SCacheObj *cache;
} SShowMgmt;
typedef struct {
int32_t connId;
SCacheObj *cache;
} SProfileMgmt;
typedef struct SMnode {
int32_t dnodeId;
int32_t clusterId;
......@@ -45,6 +62,8 @@ typedef struct SMnode {
SSdb *pSdb;
SDnode *pDnode;
SArray *pSteps;
SShowMgmt showMgmt;
SProfileMgmt profileMgmt;
MndMsgFp msgFp[TSDB_MSG_TYPE_MAX];
SendMsgToDnodeFp sendMsgToDnodeFp;
SendMsgToMnodeFp sendMsgToMnodeFp;
......@@ -53,6 +72,7 @@ typedef struct SMnode {
int32_t sver;
int32_t statusInterval;
int32_t mnodeEqualVnodeNum;
int32_t shellActivityTimer;
char *timezone;
char *locale;
char *charset;
......
source/dnode/mnode/impl/inc/mndMnode.h
浏览文件 @ d3861996
......@@ -25,6 +25,7 @@ extern "C" {
int32_t mndInitMnode(SMnode *pMnode);
void mndCleanupMnode(SMnode *pMnode);
bool mndIsMnode(SMnode *pMnode, int32_t dnodeId);
void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet);
#ifdef __cplusplus
}
......
source/dnode/mnode/impl/inc/mndShow.h
浏览文件 @ d3861996
......@@ -24,6 +24,10 @@ extern "C" {
int32_t mndInitShow(SMnode *pMnode);
void mndCleanupShow(SMnode *pMnode);
void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp);
void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp);
void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp);
void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow);
#ifdef __cplusplus
}
......
source/dnode/mnode/impl/inc/mndUser.h
浏览文件 @ d3861996
......@@ -24,7 +24,7 @@ extern "C" {
int32_t mndInitUser(SMnode *pMnode);
void mndCleanupUser(SMnode *pMnode);
SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName);
SUserObj *mndAcquireUser(SMnode *pMnode, char *userName);
void mndReleaseUser(SMnode *pMnode, SUserObj *pUser);
#ifdef __cplusplus
......
source/dnode/mnode/impl/src/mndAcct.c
浏览文件 @ d3861996
......@@ -14,10 +14,61 @@
*/
#define _DEFAULT_SOURCE
#include "mndInt.h"
#include "mndAcct.h"
#include "mndShow.h"
#define SDB_ACCT_VER 1
static int32_t mnodeCreateDefaultAcct(SMnode *pMnode);
static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct);
static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw);
static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct);
static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct);
static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *pDstAcct);
static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg);
static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg);
static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg);
int32_t mndInitAcct(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_ACCT,
.keyType = SDB_KEY_BINARY,
.deployFp = mnodeCreateDefaultAcct,
.encodeFp = (SdbEncodeFp)mnodeAcctActionEncode,
.decodeFp = (SdbDecodeFp)mnodeAcctActionDecode,
.insertFp = (SdbInsertFp)mnodeAcctActionInsert,
.updateFp = (SdbUpdateFp)mnodeAcctActionUpdate,
.deleteFp = (SdbDeleteFp)mnodeAcctActionDelete};
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_ACCT, mndProcessCreateAcctMsg);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_ACCT, mndProcessAlterAcctMsg);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_ACCT, mndProcessDropAcctMsg);
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupAcct(SMnode *pMnode) {}
static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) {
SAcctObj acctObj = {0};
tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN);
acctObj.createdTime = taosGetTimestampMs();
acctObj.updateTime = acctObj.createdTime;
acctObj.acctId = 1;
acctObj.cfg = (SAcctCfg){.maxUsers = 1024,
.maxDbs = 1024,
.maxTimeSeries = INT32_MAX,
.maxStreams = 8092,
.maxStorage = INT64_MAX,
.accessState = TSDB_VN_ALL_ACCCESS};
SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj);
if (pRaw == NULL) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mTrace("acct:%s, will be created while deploy sdb", acctObj.acct);
return sdbWrite(pMnode->pSdb, pRaw);
}
static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj));
if (pRaw == NULL) return NULL;
......@@ -92,40 +143,20 @@ static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *p
return 0;
}
static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) {
int32_t code = 0;
SAcctObj acctObj = {0};
tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN);
acctObj.createdTime = taosGetTimestampMs();
acctObj.updateTime = acctObj.createdTime;
acctObj.acctId = 1;
acctObj.cfg = (SAcctCfg){.maxUsers = 1024,
.maxDbs = 1024,
.maxTimeSeries = INT32_MAX,
.maxStreams = 8092,
.maxStorage = INT64_MAX,
.accessState = TSDB_VN_ALL_ACCCESS};
SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj);
if (pRaw == NULL) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mTrace("acct:%s, will be created while deploy sdb", acctObj.acct);
return sdbWrite(pMnode->pSdb, pRaw);
static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg) {
terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED;
mError("failed to process create acct msg since %s", terrstr());
return -1;
}
int32_t mndInitAcct(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_ACCT,
.keyType = SDB_KEY_BINARY,
.deployFp = mnodeCreateDefaultAcct,
.encodeFp = (SdbEncodeFp)mnodeAcctActionEncode,
.decodeFp = (SdbDecodeFp)mnodeAcctActionDecode,
.insertFp = (SdbInsertFp)mnodeAcctActionInsert,
.updateFp = (SdbUpdateFp)mnodeAcctActionUpdate,
.deleteFp = (SdbDeleteFp)mnodeAcctActionDelete};
return sdbSetTable(pMnode->pSdb, table);
static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg) {
terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED;
mError("failed to process create acct msg since %s", terrstr());
return -1;
}
void mndCleanupAcct(SMnode *pMnode) {}
static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg) {
terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED;
mError("failed to process create acct msg since %s", terrstr());
return -1;
}
\ No newline at end of file
source/dnode/mnode/impl/src/mndCluster.c
浏览文件 @ d3861996
......@@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "mndCluster.h"
#include "mndTrans.h"
#include "mndShow.h"
#define SDB_CLUSTER_VER 1
......@@ -94,6 +95,71 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
return sdbWrite(pMnode->pSdb, pRaw);
}
// static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
// int32_t cols = 0;
// SSchema *pSchema = pMeta->schema;
// pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE;
// pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
// strcpy(pSchema[cols].name, "clusterId");
// pSchema[cols].bytes = htons(pShow->bytes[cols]);
// cols++;
// pShow->bytes[cols] = 8;
// pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
// strcpy(pSchema[cols].name, "create_time");
// pSchema[cols].bytes = htons(pShow->bytes[cols]);
// cols++;
// pMeta->numOfColumns = htons(cols);
// strcpy(pMeta->tableFname, "show cluster");
// pShow->numOfColumns = cols;
// pShow->offset[0] = 0;
// for (int32_t i = 1; i < cols; ++i) {
// pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
// }
// pShow->numOfRows = 1;
// pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
// return 0;
// }
// static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
// int32_t numOfRows = 0;
// int32_t cols = 0;
// char * pWrite;
// SClusterObj *pCluster = NULL;
// while (numOfRows < rows) {
// pShow->pIter = mnodeGetNextCluster(pShow->pIter, &pCluster);
// if (pCluster == NULL) break;
// cols = 0;
// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
// STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pCluster->uid, TSDB_CLUSTER_ID_LEN);
// cols++;
// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
// *(int64_t *) pWrite = pCluster->createdTime;
// cols++;
// mnodeDecClusterRef(pCluster);
// numOfRows++;
// }
// mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
// pShow->numOfReads += numOfRows;
// return numOfRows;
// }
// static void mnodeCancelGetNextCluster(void *pIter) {
// sdbFreeIter(tsClusterSdb, pIter);
// }
int32_t mndInitCluster(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_CLUSTER,
.keyType = SDB_KEY_INT32,
......@@ -104,6 +170,9 @@ int32_t mndInitCluster(SMnode *pMnode) {
.updateFp = (SdbUpdateFp)mndClusterActionUpdate,
.deleteFp = (SdbDeleteFp)mndClusterActionDelete};
// mndAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta);
// mndAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters);
// mndAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster);
return sdbSetTable(pMnode->pSdb, table);
}
......
source/dnode/mnode/impl/src/mndDb.c
浏览文件 @ d3861996
......@@ -14,8 +14,39 @@
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "mndInt.h"
#include "mndDb.h"
static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg);
int32_t mndInitDb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mnodeProcessUseMsg);
return 0;
}
int32_t mndInitDb(SMnode *pMnode) { return 0; }
void mndCleanupDb(SMnode *pMnode) {}
SDbObj *mndAcquireDb(SMnode *pMnode, char *db) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_DB, db);
}
void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pDb);
}
static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SUseDbMsg *pUse = pMsg->rpcMsg.pCont;
strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN);
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
if (pDb != NULL) {
mndReleaseDb(pMnode, pDb);
return 0;
} else {
mError("db:%s, failed to process use db msg since %s", pMsg->db, terrstr());
return -1;
}
}
source/dnode/mnode/impl/src/mndDnode.c
浏览文件 @ d3861996
......@@ -226,7 +226,8 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) {
pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime);
}
static int32_t mndProcessStatusMsg(SMnode *pMnode, SMnodeMsg *pMsg) {
static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SStatusMsg *pStatus = pMsg->rpcMsg.pCont;
mndParseStatusMsg(pStatus);
......@@ -315,11 +316,11 @@ static int32_t mndProcessStatusMsg(SMnode *pMnode, SMnodeMsg *pMsg) {
return 0;
}
static int32_t mndProcessCreateDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessConfigDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { return 0; }
int32_t mndInitDnode(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_DNODE,
......
source/dnode/mnode/impl/src/mndMnode.c
浏览文件 @ d3861996
......@@ -103,9 +103,9 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
return sdbWrite(pMnode->pSdb, pRaw);
}
static int32_t mndProcessCreateMnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropMnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { return 0; }
int32_t mndInitMnode(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_MNODE,
......@@ -136,3 +136,5 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) {
sdbRelease(pSdb, pMnodeObj);
return true;
}
void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {}
\ No newline at end of file
source/dnode/mnode/impl/src/mndShow.c
浏览文件 @ d3861996
......@@ -14,8 +14,322 @@
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "mndInt.h"
#include "mndShow.h"
int32_t mndInitShow(SMnode *pMnode) { return 0; }
void mndCleanupShow(SMnode *pMnode) {}
\ No newline at end of file
static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg);
static int32_t mndProcessRetrieveMsg( SMnodeMsg *pMsg);
static bool mndCheckRetrieveFinished(SShowObj *pShow);
static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow);
static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove);
static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow);
static void mndFreeShowObj(void *ppShow);
static char *mndShowStr(int32_t showType);
int32_t mndInitShow(SMnode *pMnode) {
SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->cache = taosCacheInit(TSDB_CACHE_PTR_KEY, 5, true, mndFreeShowObj, "show");
if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc show cache since %s", terrstr());
return -1;
}
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SHOW, mndProcessShowMsg);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SHOW_RETRIEVE, mndProcessRetrieveMsg);
return 0;
}
void mndCleanupShow(SMnode *pMnode) {
SShowMgmt *pMgmt = &pMnode->showMgmt;
if (pMgmt->cache != NULL) {
taosCacheCleanup(pMgmt->cache);
pMgmt->cache = NULL;
}
}
static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow) {
TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pShow;
SShowMgmt *pMgmt = &pMnode->showMgmt;
SShowObj **ppShow = taosCacheAcquireByKey(pMgmt->cache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE));
if (ppShow) {
mTrace("show:%d, data:%p acquired from cache", pShow->id, ppShow);
return 0;
}
return -1;
}
static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) {
SMnode *pMnode = pShow->pMnode;
SShowMgmt *pMgmt = &pMnode->showMgmt;
SShowObj **ppShow = (SShowObj **)pShow->ppShow;
taosCacheRelease(pMgmt->cache, (void **)(&ppShow), forceRemove);
mDebug("show:%d, data:%p released from cache, force:%d", pShow->id, ppShow, forceRemove);
}
static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow) {
SShowMgmt *pMgmt = &pMnode->showMgmt;
int32_t lifeSpan = pMnode->shellActivityTimer * 6 * 1000;
TSDB_CACHE_PTR_TYPE val = (TSDB_CACHE_PTR_TYPE)pShow;
pShow->id = atomic_add_fetch_32(&pMgmt->showId, 1);
SShowObj **ppShow =
taosCachePut(pMgmt->cache, &val, sizeof(TSDB_CACHE_PTR_TYPE), &pShow, sizeof(TSDB_CACHE_PTR_TYPE), lifeSpan);
if (ppShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:%d, failed to put into cache", pShow->id);
return -1;
}
mTrace("show:%d, data:%p put into cache", pShow->id, ppShow);
return 0;
}
static void mndFreeShowObj(void *ppShow) {
SShowObj *pShow = *(SShowObj **)ppShow;
SMnode *pMnode = pShow->pMnode;
SShowMgmt *pMgmt = &pMnode->showMgmt;
ShowFreeIterFp freeFp = pMgmt->freeIterFps[pShow->type];
if (freeFp != NULL) {
if (pShow->pVgIter != NULL) {
// only used in 'show vnodes "ep"'
(*freeFp)(pMnode, pShow->pVgIter);
}
if (pShow->pIter != NULL) {
(*freeFp)(pMnode, pShow->pIter);
}
}
mDebug("show:%d, data:%p destroyed", pShow->id, ppShow);
tfree(pShow);
}
static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) {
SMnode *pMnode = pMnodeMsg->pMnode;
SShowMgmt *pMgmt = &pMnode->showMgmt;
SShowMsg *pMsg = pMnodeMsg->rpcMsg.pCont;
int8_t type = pMsg->type;
uint16_t payloadLen = htonl(pMsg->payloadLen);
if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) {
terrno = TSDB_CODE_MND_INVALID_MSG_TYPE;
mError("failed to process show msg since %s", terrstr());
return -1;
}
ShowMetaFp metaFp = pMgmt->metaFps[type];
if (metaFp == NULL) {
terrno = TSDB_CODE_MND_INVALID_MSG_TYPE;
mError("failed to process show-meta msg:%s since no message handle", mndShowStr(type));
return -1;
}
int32_t size = sizeof(SShowObj) + payloadLen;
SShowObj *pShow = calloc(1, size);
if (pShow != NULL) {
pShow->pMnode = pMnode;
pShow->type = type;
pShow->payloadLen = payloadLen;
memcpy(pShow->db, pMsg->db, TSDB_FULL_DB_NAME_LEN);
memcpy(pShow->payload, pMsg->payload, payloadLen);
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr());
return -1;
}
if (mndPutShowObj(pMnode, pShow) == 0) {
mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr());
free(pShow);
return -1;
}
size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
SShowRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {
mndReleaseShowObj(pShow, true);
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:%d, failed to process show-meta msg:%s since malloc rsp error", pShow->id, mndShowStr(type));
return -1;
}
pRsp->qhandle = htobe64((uint64_t)pShow);
int32_t code = (*metaFp)(pMnodeMsg,pShow, &pRsp->tableMeta);
mDebug("show:%d, type:%s, get meta finished, numOfRows:%d cols:%d result:%s", pShow->id, mndShowStr(type),
pShow->numOfRows, pShow->numOfColumns, tstrerror(code));
if (code == TSDB_CODE_SUCCESS) {
pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
pMnodeMsg->pCont = pRsp;
mndReleaseShowObj(pShow, false);
return TSDB_CODE_SUCCESS;
} else {
rpcFreeCont(pRsp);
mndReleaseShowObj(pShow, true);
return code;
}
}
static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
SMnode *pMnode = pMnodeMsg->pMnode;
SShowMgmt *pMgmt = &pMnode->showMgmt;
int32_t rowsToRead = 0;
int32_t size = 0;
int32_t rowsRead = 0;
SRetrieveTableMsg *pRetrieve = pMnodeMsg->rpcMsg.pCont;
pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
SShowObj *pShow = (SShowObj *)pRetrieve->qhandle;
/*
* in case of server restart, apps may hold qhandle created by server before
* restart, which is actually invalid, therefore, signature check is required.
*/
if (mndAcquireShowObj(pMnode, pShow) != 0) {
terrno = TSDB_CODE_MND_INVALID_SHOWOBJ;
mError("failed to process show-retrieve msg:%p since %s", pShow, terrstr());
return -1;
}
ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
if (retrieveFp == NULL) {
mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr());
return -1;
}
mDebug("show:%d, type:%s, start retrieve data, numOfReads:%d numOfRows:%d", pShow->id, mndShowStr(pShow->type),
pShow->numOfReads, pShow->numOfRows);
if (mndCheckRetrieveFinished(pShow)) {
mDebug("show:%d, read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows);
pShow->numOfReads = pShow->numOfRows;
}
if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
rowsToRead = pShow->numOfRows - pShow->numOfReads;
}
/* return no more than 100 tables in one round trip */
if (rowsToRead > 100) rowsToRead = 100;
/*
* the actual number of table may be larger than the value of pShow->numOfRows, if a query is
* issued during a continuous create table operation. Therefore, rowToRead may be less than 0.
*/
if (rowsToRead < 0) rowsToRead = 0;
size = pShow->rowSize * rowsToRead;
size += 100;
SRetrieveTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {
mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr());
return -1;
}
// if free flag is set, client wants to clean the resources
if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead);
}
mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead);
pRsp->numOfRows = htonl(rowsRead);
pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision
pMnodeMsg->pCont = pRsp;
pMnodeMsg->contLen = size;
if (rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) {
pRsp->completed = 1;
mDebug("%p, retrieve completed", pShow);
mndReleaseShowObj(pShow, true);
} else {
mDebug("%p, retrieve not completed yet", pShow);
mndReleaseShowObj(pShow, false);
}
return TSDB_CODE_SUCCESS;
}
static char *mndShowStr(int32_t showType) {
switch (showType) {
case TSDB_MGMT_TABLE_ACCT:
return "show accounts";
case TSDB_MGMT_TABLE_USER:
return "show users";
case TSDB_MGMT_TABLE_DB:
return "show databases";
case TSDB_MGMT_TABLE_TABLE:
return "show tables";
case TSDB_MGMT_TABLE_DNODE:
return "show dnodes";
case TSDB_MGMT_TABLE_MNODE:
return "show mnodes";
case TSDB_MGMT_TABLE_VGROUP:
return "show vgroups";
case TSDB_MGMT_TABLE_METRIC:
return "show stables";
case TSDB_MGMT_TABLE_MODULE:
return "show modules";
case TSDB_MGMT_TABLE_QUERIES:
return "show queries";
case TSDB_MGMT_TABLE_STREAMS:
return "show streams";
case TSDB_MGMT_TABLE_VARIABLES:
return "show configs";
case TSDB_MGMT_TABLE_CONNS:
return "show connections";
case TSDB_MGMT_TABLE_SCORES:
return "show scores";
case TSDB_MGMT_TABLE_GRANTS:
return "show grants";
case TSDB_MGMT_TABLE_VNODES:
return "show vnodes";
case TSDB_MGMT_TABLE_CLUSTER:
return "show clusters";
case TSDB_MGMT_TABLE_STREAMTABLES:
return "show streamtables";
case TSDB_MGMT_TABLE_TP:
return "show topics";
default:
return "undefined";
}
}
static bool mndCheckRetrieveFinished(SShowObj *pShow) {
if (pShow->pIter == NULL && pShow->numOfReads != 0) {
return true;
}
return false;
}
void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) {
if (rows < capacity) {
for (int32_t i = 0; i < numOfCols; ++i) {
memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows);
}
}
}
void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp) {
SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->metaFps[showType] = fp;
}
void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp) {
SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->retrieveFps[showType] = fp;
}
void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType showType, ShowFreeIterFp fp) {
SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->freeIterFps[showType] = fp;
}
source/dnode/mnode/impl/src/mndUser.c
浏览文件 @ d3861996
......@@ -14,12 +14,76 @@
*/
#define _DEFAULT_SOURCE
#include "mndUser.h"
#include "mndShow.h"
#include "mndSync.h"
#include "mndTrans.h"
#include "tkey.h"
#define SDB_USER_VER 1
static int32_t mndCreateDefaultUsers(SMnode *pMnode);
static SSdbRaw *mndUserActionEncode(SUserObj *pUser);
static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw);
static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser);
static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser);
static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser);
static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg);
static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg);
static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg);
static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg);
int32_t mndInitUser(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_USER,
.keyType = SDB_KEY_BINARY,
.deployFp = (SdbDeployFp)mndCreateDefaultUsers,
.encodeFp = (SdbEncodeFp)mndUserActionEncode,
.decodeFp = (SdbDecodeFp)mndUserActionDecode,
.insertFp = (SdbInsertFp)mndUserActionInsert,
.updateFp = (SdbUpdateFp)mndUserActionUpdate,
.deleteFp = (SdbDeleteFp)mndUserActionDelete};
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_USER, mndProcessCreateUserMsg);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_USER, mndProcessAlterUserMsg);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_USER, mndProcessDropUserMsg);
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupUser(SMnode *pMnode) {}
static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) {
SUserObj userObj = {0};
tstrncpy(userObj.user, user, TSDB_USER_LEN);
tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass);
userObj.createdTime = taosGetTimestampMs();
userObj.updateTime = userObj.createdTime;
if (strcmp(user, TSDB_DEFAULT_USER) == 0) {
userObj.superAuth = 1;
}
SSdbRaw *pRaw = mndUserActionEncode(&userObj);
if (pRaw == NULL) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mTrace("user:%s, will be created while deploy sdb", userObj.user);
return sdbWrite(pMnode->pSdb, pRaw);
}
static int32_t mndCreateDefaultUsers(SMnode *pMnode) {
if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) {
return -1;
}
if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) {
return -1;
}
return 0;
}
static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, SDB_USER_VER, sizeof(SUserObj));
if (pRaw == NULL) return NULL;
......@@ -30,7 +94,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN)
SDB_SET_INT64(pRaw, dataPos, pUser->createdTime)
SDB_SET_INT64(pRaw, dataPos, pUser->updateTime)
SDB_SET_INT8(pRaw, dataPos, pUser->rootAuth)
SDB_SET_INT8(pRaw, dataPos, pUser->superAuth)
SDB_SET_DATALEN(pRaw, dataPos);
return pRaw;
......@@ -56,7 +120,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->acct, TSDB_USER_LEN)
SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime)
SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->rootAuth)
SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superAuth)
return pRow;
}
......@@ -70,12 +134,14 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) {
return -1;
}
pUser->pAcct = sdbAcquire(pSdb, SDB_ACCT, pUser->acct);
if (pUser->pAcct == NULL) {
SAcctObj *pAcct = sdbAcquire(pSdb, SDB_ACCT, pUser->acct);
if (pAcct == NULL) {
terrno = TSDB_CODE_MND_ACCT_NOT_EXIST;
mError("user:%s, failed to perform insert action since %s", pUser->user, terrstr());
return -1;
}
pUser->acctId = pAcct->acctId;
sdbRelease(pSdb, pAcct);
return 0;
}
......@@ -87,11 +153,6 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
pUser->prohibitDbHash = NULL;
}
if (pUser->pAcct != NULL) {
sdbRelease(pSdb, pUser->pAcct);
pUser->pAcct = NULL;
}
return 0;
}
......@@ -102,40 +163,18 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDs
memcpy(pSrcUser->acct, pDstUser->acct, TSDB_USER_LEN);
pSrcUser->createdTime = pDstUser->createdTime;
pSrcUser->updateTime = pDstUser->updateTime;
pSrcUser->rootAuth = pDstUser->rootAuth;
pSrcUser->superAuth = pDstUser->superAuth;
return 0;
}
static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) {
SUserObj userObj = {0};
tstrncpy(userObj.user, user, TSDB_USER_LEN);
tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass);
userObj.createdTime = taosGetTimestampMs();
userObj.updateTime = userObj.createdTime;
if (strcmp(user, TSDB_DEFAULT_USER) == 0) {
userObj.rootAuth = 1;
}
SSdbRaw *pRaw = mndUserActionEncode(&userObj);
if (pRaw == NULL) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mTrace("user:%s, will be created while deploy sdb", userObj.user);
return sdbWrite(pMnode->pSdb, pRaw);
SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_USER, userName);
}
static int32_t mndCreateDefaultUsers(SMnode *pMnode) {
if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) {
return -1;
}
if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) {
return -1;
}
return 0;
void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pUser);
}
static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg) {
......@@ -145,7 +184,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass,
taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass);
userObj.createdTime = taosGetTimestampMs();
userObj.updateTime = userObj.createdTime;
userObj.rootAuth = 0;
userObj.superAuth = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) return -1;
......@@ -183,7 +222,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass,
return 0;
}
static int32_t mndProcessCreateUserMsg(SMnode *pMnode, SMnodeMsg *pMsg) {
static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
if (pCreate->user[0] == 0) {
......@@ -224,30 +264,14 @@ static int32_t mndProcessCreateUserMsg(SMnode *pMnode, SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
int32_t mndInitUser(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_USER,
.keyType = SDB_KEY_BINARY,
.deployFp = (SdbDeployFp)mndCreateDefaultUsers,
.encodeFp = (SdbEncodeFp)mndUserActionEncode,
.decodeFp = (SdbDecodeFp)mndUserActionDecode,
.insertFp = (SdbInsertFp)mndUserActionInsert,
.updateFp = (SdbUpdateFp)mndUserActionUpdate,
.deleteFp = (SdbDeleteFp)mndUserActionDelete};
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_USER, mndProcessCreateUserMsg);
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupUser(SMnode *pMnode) {}
SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_USER, &userName);
static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) {
terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED;
mError("failed to process alter user msg since %s", terrstr());
return -1;
}
void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pUser);
static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) {
terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED;
mError("failed to process drop user msg since %s", terrstr());
return -1;
}
\ No newline at end of file
source/dnode/mnode/impl/src/mnode.c
浏览文件 @ d3861996
......@@ -206,6 +206,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
pMnode->sver = pOption->sver;
pMnode->statusInterval = pOption->statusInterval;
pMnode->mnodeEqualVnodeNum = pOption->mnodeEqualVnodeNum;
pMnode->shellActivityTimer = pOption->shellActivityTimer;
pMnode->timezone = strdup(pOption->timezone);
pMnode->locale = strdup(pOption->locale);
pMnode->charset = strdup(pOption->charset);
......@@ -386,7 +387,7 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) {
goto PROCESS_RPC_END;
}
code = (*fp)(pMnode, pMsg);
code = (*fp)(pMsg);
if (code != 0) {
code = terrno;
mError("msg:%p, app:%p failed to process since %s", pMsg, ahandle, terrstr());
......
source/libs/index/inc/index_fst.h
浏览文件 @ d3861996
......@@ -16,6 +16,9 @@
#ifndef __INDEX_FST_H__
#define __INDEX_FST_H__
#ifdef __cplusplus
extern "C" {
#endif
#include "tarray.h"
#include "index_fst_util.h"
......@@ -34,6 +37,7 @@ typedef struct FstRange {
} FstRange;
typedef enum {GE, GT, LE, LT} RangeType;
typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State;
typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType;
......@@ -93,8 +97,11 @@ typedef struct FstBuilder {
FstBuilder *fstBuilderCreate(void *w, FstType ty);
void fstBuilderDestroy(FstBuilder *b);
void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in);
bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in);
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup);
void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate);
CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn);
......@@ -169,11 +176,6 @@ uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null
#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans)
......@@ -272,7 +274,6 @@ FstNode* fstGetNode(Fst *fst, CompiledAddr);
FstNode* fstGetRoot(Fst *fst);
FstType fstGetType(Fst *fst);
CompiledAddr fstGetRootAddr(Fst *fst);
Output fstEmptyFinalOutput(Fst *fst, bool *null);
bool fstVerify(Fst *fst);
......@@ -280,10 +281,6 @@ bool fstVerify(Fst *fst);
//refactor this function
bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
typedef struct StreamState {
FstNode *node;
uint64_t trans;
......@@ -310,10 +307,30 @@ typedef struct StreamWithStateResult {
} StreamWithStateResult;
StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state);
void swsResultDestroy(StreamWithStateResult *result);
typedef void* (*StreamCallback)(void *);
StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) ;
void streamWithStateDestroy(StreamWithState *sws);
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min);
StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback);
typedef struct FstStreamBuilder {
Fst *fst;
Automation *aut;
FstBoundWithData *min;
FstBoundWithData *max;
} FstStreamBuilder;
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut);
// set up bound range
// refator, simple code by marco
FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type);
#ifdef __cplusplus
}
#endif
#endif
source/libs/index/inc/index_fst_automation.h
浏览文件 @ d3861996
......@@ -15,6 +15,10 @@
#ifndef __INDEX_FST_AUTAOMATION_H__
#define __INDEX_FST_AUTAOMATION_H__
#ifdef __cplusplus
extern "C" {
#endif
typedef struct AutomationCtx AutomationCtx;
typedef struct StartWith {
......@@ -42,6 +46,8 @@ typedef struct Automation {
void *data;
} Automation;
#ifdef __cplusplus
}
#endif
#endif
source/libs/index/inc/index_fst_common.h
浏览文件 @ d3861996
#ifndef __INDEX_FST_COMM_H__
#define __INDEX_FST_COMM_H__
extern const uint8_t COMMON_INPUTS[];
extern char const COMMON_INPUTS_INV[];
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif
source/libs/index/inc/index_fst_counting_writer.h
浏览文件 @ d3861996
......@@ -16,6 +16,38 @@
#ifndef __INDEX_FST_COUNTING_WRITER_H__
#define __INDEX_FST_COUNTING_WRITER_H__
#ifdef __cplusplus
extern "C" {
#endif
#include "tfile.h"
#define DefaultMem 1024*1024
static char tmpFile[] = "/tmp/index";
typedef enum WriterType {TMemory, TFile} WriterType;
typedef struct WriterCtx {
int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len);
int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len);
int (*flush)(struct WriterCtx *ctx);
WriterType type;
union {
int fd;
void *mem;
};
int32_t offset;
int32_t limit;
} WriterCtx;
static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len);
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len);
static int writeCtxDoFlush(WriterCtx *ctx);
WriterCtx* writerCtxCreate(WriterType type);
void writerCtxDestroy(WriterCtx *w);
typedef uint32_t CheckSummer;
......@@ -25,7 +57,7 @@ typedef struct FstCountingWriter {
CheckSummer summer;
} FstCountingWriter;
uint64_t fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen);
int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen);
int fstCountingWriterFlush(FstCountingWriter *write);
......@@ -44,6 +76,10 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n);
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
#ifdef __cplusplus
}
#endif
#endif
source/libs/index/inc/index_fst_node.h
浏览文件 @ d3861996
......@@ -16,6 +16,10 @@
#ifndef __INDEX_FST_NODE_H__
#define __INDEX_FST_NODE_H__
#ifdef __cplusplus
extern "C" {
#endif
#include "index_fst_util.h"
#include "index_fst_counting_writer.h"
......@@ -42,7 +46,12 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src);
void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src);
//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2);
void fstBuilderNodeDestroy(FstBuilderNode *node);
#ifdef __cplusplus
}
#endif
#endif
source/libs/index/inc/index_fst_registry.h
浏览文件 @ d3861996
......@@ -15,6 +15,10 @@
#ifndef __FST_REGISTRY_H__
#define __FST_REGISTRY_H__
#ifdef __cplusplus
extern "C" {
#endif
#include "index_fst_util.h"
#include "tarray.h"
#include "index_fst_node.h"
......@@ -59,4 +63,8 @@ void fstRegistryDestroy(FstRegistry *registry);
FstRegistryEntry* fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode);
void fstRegistryEntryDestroy(FstRegistryEntry *entry);
#ifdef __cplusplus
}
#endif
#endif
source/libs/index/inc/index_fst_util.h
浏览文件 @ d3861996
......@@ -17,6 +17,10 @@
#ifndef __INDEX_FST_UTIL_H__
#define __INDEX_FST_UTIL_H__
#ifdef __cplusplus
extern "C" {
#endif
#include "tarray.h"
#include "index_fst_common.h"
......@@ -67,20 +71,30 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr);
CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr);
typedef struct FstString {
uint8_t *data;
uint32_t len;
int32_t ref;
} FstString;
typedef struct FstSlice {
uint8_t *data;
uint64_t dLen;
FstString *str;
int32_t start;
int32_t end;
} FstSlice;
FstSlice fstSliceCopy(FstSlice *slice, int32_t start, int32_t end);
FstSlice fstSliceCreate(uint8_t *data, uint64_t dLen);
bool fstSliceEmpty(FstSlice *slice);
int fstSliceCompare(FstSlice *a, FstSlice *b);
FstSlice fstSliceCreate(uint8_t *data, uint64_t len);
FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end);
FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end);
bool fstSliceIsEmpty(FstSlice *s);
int fstSliceCompare(FstSlice *s1, FstSlice *s2);
void fstSliceDestroy(FstSlice *s);
uint8_t *fstSliceData(FstSlice *s, int32_t *sz);
#define FST_SLICE_LEN(s) ((s)->end - (s)->start + 1)
#define FST_SLICE_LEN(s) (s->end - s->start + 1)
#ifdef __cplusplus
}
#endif
#endif
source/libs/index/src/index_fst.c
浏览文件 @ d3861996
......@@ -52,7 +52,7 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) {
FstBuilderNode *node = malloc(sizeof(FstBuilderNode));
node->isFinal = isFinal;
node->finalOutput = 0;
node->trans = NULL;
node->trans = taosArrayInit(16, sizeof(FstTransition));
FstBuilderNodeUnfinished un = {.node = node, .last = NULL};
taosArrayPush(nodes->stack, &un);
......@@ -62,7 +62,7 @@ FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) {
assert(taosArrayGetSize(nodes->stack) == 1);
FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack);
assert(un->last == NULL);
//assert(un->last == NULL);
return un->node;
}
......@@ -70,6 +70,7 @@ FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledA
FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack);
fstBuilderNodeUnfinishedLastCompiled(un, addr);
free(un->last); // TODO add func FstLastTransitionFree()
un->last = NULL;
return un->node;
}
......@@ -92,30 +93,30 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr add
}
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output out) {
FstSlice *s = &bs;
if (s->data == NULL || s->dLen == 0 || s->start > s->end) {
if (fstSliceIsEmpty(s)) {
return;
}
size_t sz = taosArrayGetSize(nodes->stack) - 1;
FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz);
assert(un->last == NULL);
//FstLastTransition *trn = malloc(sizeof(FstLastTransition));
//trn->inp = s->data[s->start];
//trn->out = out;
un->last = fstLastTransitionCreate(s->data[s->start], out);
int32_t len = 0;
uint8_t *data = fstSliceData(s, &len);
un->last = fstLastTransitionCreate(data[0], out);
for (uint64_t i = s->start; i <= s->end; i++) {
for (uint64_t i = 1; i < len; i++) {
FstBuilderNode *n = malloc(sizeof(FstBuilderNode));
n->isFinal = false;
n->finalOutput = 0;
n->trans = NULL;
n->trans = taosArrayInit(16, sizeof(FstTransition));
//FstLastTransition *trn = malloc(sizeof(FstLastTransition));
//trn->inp = s->data[i];
//trn->out = out;
FstLastTransition *trn = fstLastTransitionCreate(s->data[i], out);
FstLastTransition *trn = fstLastTransitionCreate(data[i], 0);
FstBuilderNodeUnfinished un = {.node = n, .last = trn};
taosArrayPush(nodes->stack, &un);
......@@ -127,13 +128,13 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) {
FstSlice *s = &bs;
size_t lsz = (size_t)(s->end - s->start + 1); // data len
size_t ssz = taosArrayGetSize(node->stack); // stack size
uint64_t count = 0;
int32_t lsz; // data len
uint8_t *data = fstSliceData(s, &lsz);
for (size_t i = 0; i < ssz && i < lsz; i++) {
FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i);
if (un->last->inp == s->data[s->start + i]) {
if (un->last->inp == data[i]) {
count++;
} else {
break;
......@@ -153,7 +154,8 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node,
FstLastTransition *t = un->last;
uint64_t addPrefix = 0;
if (t && t->inp == s->data[s->start + i]) {
uint8_t *data = fstSliceData(s, NULL);
if (t && t->inp == data[i]) {
uint64_t commPrefix = MIN(t->out, *out);
uint64_t tAddPrefix = t->out - commPrefix;
(*out) = (*out) - commPrefix;
......@@ -164,7 +166,6 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node,
}
if (addPrefix != 0) {
fstBuilderNodeUnfinishedAddOutputPrefix(un, addPrefix);
}
}
return i;
......@@ -176,7 +177,9 @@ FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) {
if (addr == EMPTY_ADDRESS) {
return fs;
}
uint8_t v = slice->data[addr];
uint8_t *data = fstSliceData(slice, NULL);
uint8_t v = data[addr];
uint8_t t = (v & 0b11000000) >> 6;
if (t == 0b11) {
fs.state = OneTransNext;
......@@ -376,7 +379,8 @@ uint8_t fstStateInput(FstState *s, FstNode *node) {
FstSlice *slice = &node->data;
bool null = false;
uint8_t inp = fstStateCommInput(s, &null);
return null == false ? inp : slice->data[slice->start - 1];
uint8_t *data = fstSliceData(slice, NULL);
return null == false ? inp : data[-1];
}
uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
assert(s->state == AnyTrans);
......@@ -388,7 +392,9 @@ uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
- fstStateTransIndexSize(s, node->version, node->nTrans)
- i
- 1; // the output size
return slice->data[at];
uint8_t *data = fstSliceData(slice, NULL);
return data[at];
}
// trans_addr
......@@ -406,7 +412,8 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) {
- tSizes;
// refactor error logic
return unpackDelta(slice->data + slice->start + i, tSizes, node->end);
uint8_t *data = fstSliceData(slice, NULL);
return unpackDelta(data +i, tSizes, node->end);
}
}
CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
......@@ -421,7 +428,8 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i
- node->nTrans
- (i * tSizes)
- tSizes;
return unpackDelta(slice->data + slice->start + at, tSizes, node->end);
uint8_t *data = fstSliceData(slice, NULL);
return unpackDelta(data + at, tSizes, node->end);
}
// sizes
......@@ -434,7 +442,8 @@ PackSizes fstStateSizes(FstState *s, FstSlice *slice) {
i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1;
}
return (PackSizes)(slice->data[slice->start + i]);
uint8_t *data = fstSliceData(slice, NULL);
return (PackSizes)(*(data +i));
}
// Output
Output fstStateOutput(FstState *s, FstNode *node) {
......@@ -452,7 +461,8 @@ Output fstStateOutput(FstState *s, FstNode *node) {
- 1
- tSizes
- oSizes;
return unpackUint64(slice->data + slice->start + i, oSizes);
uint8_t *data = fstSliceData(slice, NULL);
return unpackUint64(data + i, oSizes);
}
Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
......@@ -469,7 +479,9 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
- fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans)
- (i * oSizes)
- oSizes;
return unpackUint64(slice->data + slice->start + at, oSizes);
uint8_t *data = fstSliceData(slice, NULL);
return unpackUint64(data + at, oSizes);
}
// anyTrans specify function
......@@ -523,7 +535,10 @@ uint64_t fstStateNtrans(FstState *s, FstSlice *slice) {
if (null != true) {
return n;
}
n = slice->data[slice->end - 1]; // data[data.len() - 2]
int32_t len;
uint8_t *data = fstSliceData(slice, &len);
n = data[len - 2];
//n = data[slice->end - 1]; // data[data.len() - 2]
return n == 1 ? 256: n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, then it is encoded in the state byte
}
Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) {
......@@ -538,7 +553,8 @@ Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, Pack
- fstStateTotalTransSize(s, version, sizes, nTrans)
- (nTrans * oSizes)
- oSizes;
return unpackUint64(slice->data + slice->start + at, (uint8_t)oSizes);
uint8_t *data = fstSliceData(slice, NULL);
return unpackUint64(data + at, (uint8_t)oSizes);
}
uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
......@@ -549,7 +565,10 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
- fstStateNtransLen(s)
- 1 // pack size
- fstStateTransIndexSize(s, node->version, node->nTrans);
uint64_t i = slice->data[slice->start + at + b];
int32_t dlen = 0;
uint8_t *data = fstSliceData(slice, &dlen);
uint64_t i = data[at + b];
//uint64_t i = slice->data[slice->start + at + b];
if (i >= node->nTrans) {
*null = true;
}
......@@ -561,8 +580,13 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
- node->nTrans;
uint64_t end = start + node->nTrans;
uint64_t len = end - start;
int32_t dlen = 0;
uint8_t *data = fstSliceData(slice, &dlen);
for(int i = 0; i < len; i++) {
uint8_t v = slice->data[slice->start + i];
//uint8_t v = slice->data[slice->start + i];
////slice->data[slice->start + i];
uint8_t v = data[i];
if (v == b) {
return node->nTrans - i - 1; // bug
}
......@@ -635,6 +659,7 @@ static const char *fstNodeState(FstNode *node) {
void fstNodeDestroy(FstNode *node) {
fstSliceDestroy(&node->data);
free(node);
}
FstTransitions* fstNodeTransitions(FstNode *node) {
......@@ -774,18 +799,18 @@ bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) {
void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) {
FstSlice *s = &bs;
if (fstSliceEmpty(s)) {
if (fstSliceIsEmpty(s)) {
b->len = 1;
fstUnFinishedNodesSetRootOutput(b->unfinished, in);
return;
}
Output out;
//if (in != 0) { //if let Some(in) = in
// prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out);
//} else {
// prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs);
// out = 0;
//}
Output out;
uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out);
if (prefixLen == FST_SLICE_LEN(s)) {
......@@ -798,12 +823,13 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) {
FstSlice sub = fstSliceCopy(s, prefixLen, s->end);
fstUnFinishedNodesAddSuffix(b->unfinished, sub, out);
fstSliceDestroy(&sub);
return;
}
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) {
FstSlice *input = &bs;
if (fstSliceEmpty(&b->last)) {
if (fstSliceIsEmpty(&b->last)) {
// deep copy or not
b->last = fstSliceCopy(&bs, input->start, input->end);
} else {
......@@ -829,7 +855,7 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) {
}
addr = fstBuilderCompile(b, n);
assert(addr != NONE_ADDRESS);
fstBuilderNodeDestroy(n);
//fstBuilderNodeDestroy(n);
}
fstUnFinishedNodesTopLastFreeze(b->unfinished, addr);
return;
......@@ -863,22 +889,26 @@ void* fstBuilderInsertInner(FstBuilder *b) {
FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished);
CompiledAddr rootAddr = fstBuilderCompile(b, rootNode);
uint8_t buf64[8] = {0};
char buf64[8] = {0};
taosEncodeFixedU64((void **)&buf64, b->len);
void *pBuf64 = buf64;
taosEncodeFixedU64(&pBuf64, b->len);
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
taosEncodeFixedU64((void **)&buf64, rootAddr);
pBuf64 = buf64;
taosEncodeFixedU64(&pBuf64, rootAddr);
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
uint8_t buf32[4] = {0};
char buf32[4] = {0};
void *pBuf32 = buf32;
uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt);
taosEncodeFixedU32((void **)&buf32, sum);
taosEncodeFixedU32(&pBuf32, sum);
fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32));
fstCountingWriterFlush(b->wrt);
//fstCountingWriterDestroy(b->wrt);
//b->wrt = NULL;
return b->wrt;
}
void fstBuilderFinish(FstBuilder *b) {
fstBuilderInsertInner(b);
......@@ -888,7 +918,7 @@ void fstBuilderFinish(FstBuilder *b) {
FstSlice fstNodeAsSlice(FstNode *node) {
FstSlice *slice = &node->data;
FstSlice s = fstSliceCopy(slice, slice->end, slice->dLen - 1);
FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1);
return s;
}
......@@ -929,12 +959,13 @@ void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *unNode, O
}
Fst* fstCreate(FstSlice *slice) {
char *buf = slice->data;
uint64_t skip = 0;
uint64_t len = slice->dLen;
if (len < 36) {
int32_t slen;
char *buf = fstSliceData(slice, &slen);
if (slen < 36) {
return NULL;
}
uint64_t len = slen;
uint64_t skip = 0;
uint64_t version;
taosDecodeFixedU64(buf, &version);
......@@ -992,8 +1023,10 @@ void fstDestroy(Fst *fst) {
bool fstGet(Fst *fst, FstSlice *b, Output *out) {
FstNode *root = fstGetRoot(fst);
Output tOut = 0;
for (uint32_t i = 0; i < b->dLen; i++) {
uint8_t inp = b->data[i];
int32_t len;
uint8_t *data = fstSliceData(b, &len);
for (uint32_t i = 0; i < len; i++) {
uint8_t inp = data[i];
Output res = 0;
bool null = fstNodeFindInput(root, inp, &res);
if (null) { return false; }
......@@ -1046,9 +1079,10 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) {
bool fstVerify(Fst *fst) {
uint32_t checkSum = fst->meta->checkSum;
FstSlice *data = fst->data;
int32_t len;
uint8_t *data = fstSliceData(fst->data, &len);
TSCKSUM initSum = 0;
if (!taosCheckChecksumWhole(data->data, data->dLen)) {
if (!taosCheckChecksumWhole(data, len)) {
return false;
}
return true;
......@@ -1059,8 +1093,13 @@ FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) {
FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData));
if (b == NULL) { return NULL; }
b->type = type;
if (data != NULL) {
b->data = fstSliceCopy(data, data->start, data->end);
} else {
b->data = fstSliceCreate(NULL, 0);
}
b->type = type;
return b;
}
......@@ -1078,7 +1117,7 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) {
if (bound->type == Unbounded) {
return true;
} else {
return fstSliceEmpty(&bound->data);
return fstSliceIsEmpty(&bound->data);
}
}
......@@ -1145,8 +1184,10 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
Output out = 0;
void* autState = sws->aut->start();
for (uint32_t i = 0; i < key->dLen; i++) {
uint8_t b = key->data[i];
int32_t len;
uint8_t *data = fstSliceData(key, &len);
for (uint32_t i = 0; i < len; i++) {
uint8_t b = data[i];
uint64_t res = 0;
bool null = fstNodeFindInput(node, b, &res);
if (null == false) {
......@@ -1262,12 +1303,16 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
if (fstBoundWithDataExceededBy(sws->endAt, &slice)) {
taosArrayDestroyEx(sws->stack, streamStateDestroy);
sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState));
fstSliceDestroy(&slice);
return NULL;
}
if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
FstOutput fOutput = {.null = false, out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
return swsResultCreate(&slice, fOutput , tState);
FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
StreamWithStateResult *result = swsResultCreate(&slice, fOutput , tState);
fstSliceDestroy(&slice);
return result;
}
fstSliceDestroy(&slice);
}
return NULL;
......@@ -1277,14 +1322,19 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult));
if (result == NULL) { return NULL; }
FstSlice slice = fstSliceCopy(data, 0, data->dLen - 1);
result->data = slice;
result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1);
result->out = fOut;
result->state = state;
return result;
}
void swsResultDestroy(StreamWithStateResult *result) {
if (NULL == result) { return; }
fstSliceDestroy(&result->data);
free(result);
}
void streamStateDestroy(void *s) {
if (NULL == s) { return; }
StreamState *ss = (StreamState *)s;
......@@ -1293,5 +1343,44 @@ void streamStateDestroy(void *s) {
//free(s->autoState);
}
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) {
FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder));
if (NULL == b) { return NULL; }
b->fst = fst;
b->aut = aut;
b->min = fstBoundStateCreate(Unbounded, NULL);
b->max = fstBoundStateCreate(Unbounded, NULL);
return b;
}
void fstStreamBuilderDestroy(FstStreamBuilder *b) {
fstSliceDestroy(&b->min->data);
fstSliceDestroy(&b->max->data);
free(b);
}
FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) {
if (b == NULL) { return NULL; }
if (type == GE) {
b->min->type = Included;
fstSliceDestroy(&(b->min->data));
b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1);
} else if (type == GT) {
b->min->type = Excluded;
fstSliceDestroy(&(b->min->data));
b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1);
} else if (type == LE) {
b->max->type = Included;
fstSliceDestroy(&(b->max->data));
b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1);
} else if (type == LT) {
b->max->type = Excluded;
fstSliceDestroy(&(b->max->data));
b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1);
}
return b;
}
source/libs/index/src/index_fst_automation.c
浏览文件 @ d3861996
......@@ -12,3 +12,4 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
source/libs/index/src/index_fst_counting_writer.c
浏览文件 @ d3861996
......@@ -16,24 +16,91 @@
#include "index_fst_util.h"
#include "index_fst_counting_writer.h"
static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
if (ctx->offset + len > ctx->limit) {
return -1;
}
if (ctx->type == TFile) {
assert(len != tfWrite(ctx->fd, buf, len));
} else {
memcpy(ctx->mem + ctx->offset, buf, len);
}
ctx->offset += len;
return len;
}
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) {
if (ctx->type == TFile) {
tfRead(ctx->fd, buf, len);
} else {
memcpy(buf, ctx->mem + ctx->offset, len);
}
ctx->offset += len;
return 1;
}
static int writeCtxDoFlush(WriterCtx *ctx) {
if (ctx->type == TFile) {
//tfFlush(ctx->fd);
} else {
// do nothing
}
return 1;
}
WriterCtx* writerCtxCreate(WriterType type) {
WriterCtx *ctx = calloc(1, sizeof(WriterCtx));
if (ctx == NULL) { return NULL; }
ctx->type = type;
if (ctx->type == TFile) {
ctx->fd = tfOpenCreateWriteAppend(tmpFile);
if (ctx->fd < 0) {
}
} else if (ctx->type == TMemory) {
ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t));
}
ctx->write = writeCtxDoWrite;
ctx->read = writeCtxDoRead;
ctx->flush = writeCtxDoFlush;
ctx->offset = 0;
ctx->limit = DefaultMem;
return ctx;
}
void writerCtxDestroy(WriterCtx *ctx) {
if (ctx->type == TMemory) {
free(ctx->mem);
} else {
tfClose(ctx->fd);
}
free(ctx);
}
FstCountingWriter *fstCountingWriterCreate(void *wrt) {
FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter));
if (cw == NULL) { return NULL; }
cw->wrt = wrt;
cw->wrt = (void *)(writerCtxCreate(TFile));
return cw;
}
void fstCountingWriterDestroy(FstCountingWriter *cw) {
// free wrt object: close fd or free mem
writerCtxDestroy((WriterCtx *)(cw->wrt));
free(cw);
}
uint64_t fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen) {
int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen) {
if (write == NULL) { return 0; }
// update checksum
// write data to file/socket or mem
WriterCtx *ctx = write->wrt;
write->count += bufLen;
int nWrite = ctx->write(ctx, buf, bufLen);
write->count += nWrite;
return bufLen;
}
......@@ -41,6 +108,8 @@ uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) {
return 0;
}
int fstCountingWriterFlush(FstCountingWriter *write) {
WriterCtx *ctx = write->wrt;
ctx->flush(ctx);
//write->wtr->flush
return 1;
}
......
source/libs/index/src/index_fst_node.c
浏览文件 @ d3861996
......@@ -18,7 +18,7 @@ FstBuilderNode *fstBuilderNodeDefault() {
FstBuilderNode *bn = malloc(sizeof(FstBuilderNode));
bn->isFinal = false;
bn->finalOutput = 0;
bn->trans = NULL;
bn->trans = taosArrayInit(16, sizeof(FstTransition));
return bn;
}
void fstBuilderNodeDestroy(FstBuilderNode *node) {
......@@ -27,6 +27,31 @@ void fstBuilderNodeDestroy(FstBuilderNode *node) {
taosArrayDestroy(node->trans);
free(node);
}
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) {
if (n1 == n2) { return true; }
if (n1 == NULL || n2 == NULL ) {
return false;
}
if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) {
return false;
}
size_t s1 = n1->trans? taosArrayGetSize(n1->trans): 0;
size_t s2 = n2->trans? taosArrayGetSize(n2->trans): 0;
if (s1 != s2) {
return false;
}
for (size_t i = 0; i < s1; i++) {
FstTransition *t1 = taosArrayGet(n1->trans, i);
FstTransition *t2 = taosArrayGet(n2->trans, i);
if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) {
return false;
}
}
return true;
}
FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
FstBuilderNode *node = malloc(sizeof(FstBuilderNode));
if (node == NULL) { return NULL; }
......@@ -53,12 +78,17 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
dst->isFinal = src->isFinal;
dst->finalOutput = src->finalOutput;
// avoid mem leak
//release free avoid mem leak
taosArrayDestroy(dst->trans);
dst->trans = src->trans;
src->trans = NULL;
size_t sz = taosArrayGetSize(src->trans);
dst->trans = taosArrayInit(sz, sizeof(FstTransition));
for (size_t i = 0; i < sz; i++) {
FstTransition *trn = taosArrayGet(src->trans, i);
taosArrayPush(dst->trans, trn);
}
}
//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) {
//size_t sz = taosArrayGetSize(b->trans);
......
source/libs/index/src/index_fst_registry.c
浏览文件 @ d3861996
......@@ -112,7 +112,7 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
if (end - start == 1) {
FstRegistryCell *cell = taosArrayGet(registry->table, start);
//cell->isNode &&
if (cell->addr != NONE_ADDRESS && cell->node == bNode) {
if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) {
entry->state = FOUND;
entry->addr = cell->addr ;
return entry;
......@@ -123,13 +123,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
}
} else if (end - start == 2) {
FstRegistryCell *cell1 = taosArrayGet(registry->table, start);
if (cell1->addr != NONE_ADDRESS && cell1->node == bNode) {
if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) {
entry->state = FOUND;
entry->addr = cell1->addr;
return entry;
}
FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1);
if (cell2->addr != NONE_ADDRESS && cell2->node == bNode) {
if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) {
entry->state = FOUND;
entry->addr = cell2->addr;
// must swap here
......@@ -147,7 +147,7 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
uint32_t i = start;
for (; i < end; i++) {
FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i);
if (cell->addr != NONE_ADDRESS && cell->node == bNode) {
if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) {
entry->state = FOUND;
entry->addr = cell->addr;
fstRegistryCellPromote(registry->table, i, start);
......
source/libs/index/src/index_fst_util.c
浏览文件 @ d3861996
......@@ -91,42 +91,87 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) {
}
// fst slice func
FstSlice fstSliceCreate(uint8_t *data, uint64_t dLen) {
FstSlice slice = {.data = data, .dLen = dLen, .start = 0, .end = dLen - 1};
return slice;
//
FstSlice fstSliceCreate(uint8_t *data, uint64_t len) {
FstString *str = (FstString *)malloc(sizeof(FstString));
str->ref = 1;
str->len = len;
str->data = malloc(len * sizeof(uint8_t));
memcpy(str->data, data, len);
FstSlice s = {.str = str, .start = 0, .end = len - 1};
return s;
}
// just shallow copy
FstSlice fstSliceCopy(FstSlice *slice, int32_t start, int32_t end) {
FstSlice t;
if (start >= slice->dLen || end >= slice->dLen || start > end) {
t.data = NULL;
FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) {
FstString *str = s->str;
str->ref++;
//uint8_t *buf = fstSliceData(s, &alen);
//start = buf + start - (buf - s->start);
//end = buf + end - (buf - s->start);
FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start};
return t;
};
}
FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) {
int32_t tlen = end - start + 1;
int32_t slen;
uint8_t *data = fstSliceData(s, &slen);
assert(tlen <= slen);
uint8_t *buf = malloc(sizeof(uint8_t) * tlen);
memcpy(buf, data + start, tlen);
FstString *str = malloc(sizeof(FstString));
str->data = buf;
str->len = tlen;
str->ref = 1;
FstSlice ans;
ans.str = str;
ans.start = 0;
ans.end = tlen - 1;
return ans;
}
bool fstSliceIsEmpty(FstSlice *s) {
return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0;
}
t.data = slice->data;
t.dLen = slice->dLen;
t.start = start;
t.end = end;
return t;
uint8_t *fstSliceData(FstSlice *s, int32_t *size) {
FstString *str = s->str;
if (size != NULL) {
*size = s->end - s->start + 1;
}
return str->data + s->start;
}
bool fstSliceEmpty(FstSlice *slice) {
return slice->data == NULL || slice->dLen <= 0;
void fstSliceDestroy(FstSlice *s) {
FstString *str = s->str;
str->ref--;
if (str->ref <= 0) {
free(str->data);
free(str);
s->str = NULL;
}
}
int fstSliceCompare(FstSlice *a, FstSlice *b) {
int32_t aLen = (a->end - a->start + 1);
int32_t bLen = (b->end - b->start + 1);
int32_t mLen = (aLen < bLen ? aLen : bLen);
for (int i = 0; i < mLen; i++) {
uint8_t x = a->data[i + a->start];
uint8_t y = b->data[i + b->start];
if (x == y) { continue; }
int32_t alen, blen;
uint8_t *aBuf = fstSliceData(a, &alen);
uint8_t *bBuf = fstSliceData(b, &blen);
uint32_t i, j;
for (i = 0, j = 0; i < alen && j < blen; i++, j++) {
uint8_t x = aBuf[i];
uint8_t y = bBuf[j];
if (x == y) { continue;}
else if (x < y) { return -1; }
else { return 1; }
else { return 1; };
}
if (aLen == bLen) { return 0; }
else if (aLen < bLen) { return -1; }
else { return 1; }
if (i < alen) { return 1; }
else if (j < blen) { return -1; }
else { return 0; }
}
......
source/libs/index/test/CMakeLists.txt
浏览文件 @ d3861996
add_executable(indexTest "")
target_sources(indexTest
PRIVATE
"../src/index.c"
"indexTests.cpp"
)
target_include_directories ( indexTest
......
source/libs/index/test/indexTests.cpp
浏览文件 @ d3861996
......@@ -3,58 +3,84 @@
#include <iostream>
#include "index.h"
#include "indexInt.h"
#include "index_fst.h"
#include "index_fst_util.h"
#include "index_fst_counting_writer.h"
//TEST(IndexTest, index_create_test) {
// SIndexOpts *opts = indexOptsCreate();
// SIndex *index = indexOpen(opts, "./test");
// if (index == NULL) {
// std::cout << "index open failed" << std::endl;
// }
//
//
// // write
// for (int i = 0; i < 100000; i++) {
// SIndexMultiTerm* terms = indexMultiTermCreate();
// std::string val = "field";
//
// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size());
//
// val.append(std::to_string(i));
// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size());
//
// val.insert(0, std::to_string(i));
// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size());
//
// val.append("const");
// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size());
//
//
// indexPut(index, terms, i);
// indexMultiTermDestroy(terms);
// }
//
//
// // query
// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
//
// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX);
// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM);
//
// SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
// indexSearch(index, multiQuery, result);
//
// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl;
// for (int i = 0; i < taosArrayGetSize(result); i++) {
// int *v = (int *)taosArrayGet(result, i);
// std::cout << "value --->" << *v << std::endl;
// }
// // add more test case
// indexMultiTermQueryDestroy(multiQuery);
//
// indexOptsDestroy(opts);
// indexClose(index);
// //
//}
int main(int argc, char** argv) {
std::string str("abc");
FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size());
Output val = 1;
//std::string str1("bcd");
//FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size());
//Output val2 = 10;
FstBuilder *b = fstBuilderCreate(NULL, 1);
fstBuilderInsert(b, key, val);
//fstBuilderInsert(b, key1, val2);
fstBuilderFinish(b);
fstBuilderDestroy(b);
fstSliceDestroy(&key);
return 1;
}
//TEST(IndexFstBuilder, IndexFstInput) {
//
//}
TEST(IndexTest, index_create_test) {
SIndexOpts *opts = indexOptsCreate();
SIndex *index = indexOpen(opts, "./test");
if (index == NULL) {
std::cout << "index open failed" << std::endl;
}
// write
for (int i = 0; i < 100000; i++) {
SIndexMultiTerm* terms = indexMultiTermCreate();
std::string val = "field";
indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size());
val.append(std::to_string(i));
indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size());
val.insert(0, std::to_string(i));
indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size());
val.append("const");
indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size());
indexPut(index, terms, i);
indexMultiTermDestroy(terms);
}
// query
SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX);
indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM);
SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
indexSearch(index, multiQuery, result);
std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl;
for (int i = 0; i < taosArrayGetSize(result); i++) {
int *v = (int *)taosArrayGet(result, i);
std::cout << "value --->" << *v << std::endl;
}
// add more test case
indexMultiTermQueryDestroy(multiQuery);
indexOptsDestroy(opts);
indexClose(index);
//
}
source/libs/transport/src/rpcMain.c
浏览文件 @ d3861996
......@@ -507,7 +507,6 @@ void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) {
}
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
#if 0
SRpcConn *pConn = (SRpcConn *)thandle;
if (pConn->user[0] == 0) return -1;
......@@ -516,9 +515,6 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
// pInfo->serverIp = pConn->destIp;
tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
#else
strcpy(pInfo->user, "root");
#endif
return 0;
}
......
source/libs/wal/inc/walInt.h
浏览文件 @ d3861996
......@@ -17,14 +17,16 @@
#define _TD_WAL_INT_H_
#include "wal.h"
#include "compare.h"
#ifdef __cplusplus
extern "C" {
#endif
int walRotate(SWal* pWal);
int walGetFile(SWal* pWal, int32_t version);
int64_t walGetSeq();
#ifdef __cplusplus
}
#endif
......
source/libs/wal/src/walIndex.c
浏览文件 @ d3861996
......@@ -20,9 +20,38 @@
#include "tfile.h"
#include "walInt.h"
int walSeekVerImpl(SWal *pWal, int64_t ver) {
//close old file
static int walSeekFilePos(SWal* pWal, int64_t ver) {
int code = 0;
int64_t idxTfd = pWal->curIdxTfd;
int64_t logTfd = pWal->curLogTfd;
//seek position
int64_t offset = (ver - pWal->curFileFirstVersion) * WAL_IDX_ENTRY_SIZE;
code = tfLseek(idxTfd, offset, SEEK_SET);
if(code != 0) {
}
int64_t readBuf[2];
code = tfRead(idxTfd, readBuf, sizeof(readBuf));
if(code != 0) {
}
//TODO:deserialize
ASSERT(readBuf[0] == ver);
code = tfLseek(logTfd, readBuf[1], SEEK_CUR);
if (code != 0) {
}
pWal->curLogOffset = readBuf[1];
pWal->curVersion = ver;
return code;
}
static int walChangeFile(SWal *pWal, int64_t ver) {
int code = 0;
int64_t idxTfd, logTfd;
char fnameStr[WAL_FILE_LEN];
code = tfClose(pWal->curLogTfd);
if(code != 0) {
//TODO
......@@ -32,29 +61,36 @@ int walSeekVerImpl(SWal *pWal, int64_t ver) {
//TODO
}
//bsearch in fileSet
int fName = 0;//TODO
//open the right file
char fNameStr[WAL_FILE_LEN];
sprintf(fNameStr, "%d."WAL_INDEX_SUFFIX, fName);
bool closed = 1; //TODO:read only
int64_t idxTfd = tfOpenReadWrite(fNameStr);
sprintf(fNameStr, "%d."WAL_LOG_SUFFIX, fName);
int64_t logTfd = tfOpenReadWrite(fNameStr);
//seek position
int64_t offset = (ver - fName) * WAL_IDX_ENTRY_SIZE;
tfLseek(idxTfd, offset, SEEK_SET);
//set cur version, cur file version and cur status
pWal->curFileFirstVersion = fName;
pWal->curFileLastVersion = 1;//TODO
int64_t* pRet = taosArraySearch(pWal->fileSet, &ver, compareInt64Val, TD_LE);
ASSERT(pRet != NULL);
int64_t fname = *pRet;
if(fname < pWal->lastFileName) {
pWal->curStatus &= ~WAL_CUR_FILE_WRITABLE;
pWal->curFileLastVersion = pRet[1]-1;
sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname);
idxTfd = tfOpenRead(fnameStr);
sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname);
logTfd = tfOpenRead(fnameStr);
} else {
pWal->curStatus |= WAL_CUR_FILE_WRITABLE;
pWal->curFileLastVersion = -1;
sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname);
idxTfd = tfOpenReadWrite(fnameStr);
sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname);
logTfd = tfOpenReadWrite(fnameStr);
}
pWal->curFileFirstVersion = fname;
pWal->curLogTfd = logTfd;
pWal->curIdxTfd = idxTfd;
pWal->curVersion = ver;
pWal->curOffset = offset;
pWal->curStatus = 0;//TODO
return code;
}
int walSeekVer(SWal *pWal, int64_t ver) {
if((!(pWal->curStatus & WAL_CUR_FAILED))
&& ver == pWal->curVersion) {
return 0;
}
if(ver > pWal->lastVersion) {
//TODO: some records are skipped
return -1;
......@@ -64,54 +100,13 @@ int walSeekVer(SWal *pWal, int64_t ver) {
return -1;
}
if(ver < pWal->snapshotVersion) {
//TODO: seek snapshotted log
}
if(ver >= pWal->curFileFirstVersion
&& ((pWal->curFileLastVersion == -1 && ver <= pWal->lastVersion) || (ver <= pWal->curFileLastVersion))) {
}
if(ver < pWal->curFileFirstVersion || (pWal->curFileLastVersion != -1 && ver > pWal->curFileLastVersion)) {
int index = 0;
index = 1;
//back up to avoid inconsistency
int64_t curVersion = pWal->curVersion;
int64_t curOffset = pWal->curOffset;
int64_t curFileFirstVersion = pWal->curFileFirstVersion;
int64_t curFileLastVersion = pWal->curFileLastVersion;
if(walSeekVerImpl(pWal, ver) < 0) {
//TODO: errno
pWal->curVersion = curVersion;
pWal->curOffset = curOffset;
pWal->curFileFirstVersion = curFileFirstVersion;
pWal->curFileLastVersion = curFileLastVersion;
return -1;
//TODO: seek snapshotted log, invalid in some cases
}
if(ver < pWal->curFileFirstVersion ||
(pWal->curFileLastVersion != -1 && ver > pWal->curFileLastVersion)) {
walChangeFile(pWal, ver);
}
walSeekFilePos(pWal, ver);
return 0;
}
int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
int code = 0;
//get index file
if(!tfValid(pWal->curIdxTfd)) {
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno));
}
if(pWal->curVersion != ver) {
if(walSeekVer(pWal, ver) != 0) {
//TODO: some records are skipped
return -1;
}
}
//check file checksum
//append index
return 0;
}
int walRotateIndex(SWal *pWal) {
//check file checksum
//create new file
//switch file
return 0;
}
source/libs/wal/src/walMgmt.c
浏览文件 @ d3861996
......@@ -26,57 +26,44 @@ int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId);
int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId);
int32_t walGetNewFile(SWal *pWal, int64_t *newFileId);
static pthread_mutex_t walInitLock = PTHREAD_MUTEX_INITIALIZER;
static int8_t walInited = 0;
typedef struct {
int32_t refSetId;
int32_t seq;
uint32_t seq;
int8_t stop;
int8_t inited;
pthread_t thread;
pthread_mutex_t mutex;
} SWalMgmt;
static SWalMgmt tsWal = {0};
static SWalMgmt tsWal = {0, .seq = 1};
static int32_t walCreateThread();
static void walStopThread();
static int32_t walInitObj(SWal *pWal);
static void walFreeObj(void *pWal);
int64_t walGetSeq() {
return (int64_t)atomic_load_32(&tsWal.seq);
}
int32_t walInit() {
//TODO: change to atomic
pthread_mutex_lock(&walInitLock);
if(walInited) {
pthread_mutex_unlock(&walInitLock);
return 0;
} else {
walInited = 1;
pthread_mutex_unlock(&walInitLock);
}
int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1);
if(old == 1) return 0;
int32_t code = 0;
tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj);
code = pthread_mutex_init(&tsWal.mutex, NULL);
if (code != 0) {
wError("failed to init wal mutex since %s", tstrerror(code));
return code;
}
code = walCreateThread();
int code = walCreateThread();
if (code != 0) {
wError("failed to init wal module since %s", tstrerror(code));
atomic_store_8(&tsWal.inited, 0);
return code;
}
wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId);
return code;
return 0;
}
void walCleanUp() {
walStopThread();
taosCloseRef(tsWal.refSetId);
pthread_mutex_destroy(&tsWal.mutex);
wInfo("wal module is cleaned up");
}
......@@ -92,7 +79,7 @@ static int walLoadFileset(SWal *pWal) {
char *name = ent->d_name;
name[WAL_NOSUFFIX_LEN] = 0;
//validate file name by regex matching
if(1 /* regex match */) {
if(1 /* TODO:regex match */) {
int64_t fnameInt64 = atoll(name);
taosArrayPush(pWal->fileSet, &fnameInt64);
}
......@@ -133,6 +120,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
walFreeObj(pWal);
return NULL;
}
walLoadFileset(pWal);
wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod);
......@@ -164,6 +152,9 @@ void walClose(SWal *pWal) {
pthread_mutex_lock(&pWal->mutex);
tfClose(pWal->curLogTfd);
tfClose(pWal->curIdxTfd);
taosArrayDestroy(pWal->fileSet);
pWal->fileSet = NULL;
pthread_mutex_unlock(&pWal->mutex);
taosRemoveRef(tsWal.refSetId, pWal->refId);
}
......@@ -188,6 +179,9 @@ static void walFreeObj(void *wal) {
wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal);
tfClose(pWal->curLogTfd);
tfClose(pWal->curIdxTfd);
taosArrayDestroy(pWal->fileSet);
pWal->fileSet = NULL;
pthread_mutex_destroy(&pWal->mutex);
tfree(pWal);
}
......@@ -197,7 +191,7 @@ static bool walNeedFsync(SWal *pWal) {
return false;
}
if (tsWal.seq % pWal->fsyncSeq == 0) {
if (atomic_load_32(&tsWal.seq) % pWal->fsyncSeq == 0) {
return true;
}
......@@ -206,16 +200,14 @@ static bool walNeedFsync(SWal *pWal) {
static void walUpdateSeq() {
taosMsleep(WAL_REFRESH_MS);
if (++tsWal.seq <= 0) {
tsWal.seq = 1;
}
atomic_add_fetch_32(&tsWal.seq, 1);
}
static void walFsyncAll() {
SWal *pWal = taosIterateRef(tsWal.refSetId, 0);
while (pWal) {
if (walNeedFsync(pWal)) {
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq);
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq));
int32_t code = tfFsync(pWal->curLogTfd);
if (code != 0) {
wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(code));
......@@ -226,16 +218,12 @@ static void walFsyncAll() {
}
static void *walThreadFunc(void *param) {
int stop = 0;
setThreadName("wal");
while (1) {
walUpdateSeq();
walFsyncAll();
pthread_mutex_lock(&tsWal.mutex);
stop = tsWal.stop;
pthread_mutex_unlock(&tsWal.mutex);
if (stop) break;
if (atomic_load_8(&tsWal.stop)) break;
}
return NULL;
......@@ -258,9 +246,7 @@ static int32_t walCreateThread() {
}
static void walStopThread() {
pthread_mutex_lock(&tsWal.mutex);
tsWal.stop = 1;
pthread_mutex_unlock(&tsWal.mutex);
atomic_store_8(&tsWal.stop, 1);
if (taosCheckPthreadValid(tsWal.thread)) {
pthread_join(tsWal.thread, NULL);
......
source/libs/wal/src/walWrite.c
浏览文件 @ d3861996
......@@ -26,10 +26,24 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
}
int32_t walRollback(SWal *pWal, int64_t ver) {
//TODO: ftruncate
return 0;
}
int32_t walTakeSnapshot(SWal *pWal, int64_t ver) {
pWal->snapshotVersion = ver;
//mark files safe to delete
int64_t* pRet = taosArraySearch(pWal->fileSet, &ver, compareInt64Val, TD_LE);
if(pRet != pWal->fileSet->pData) {
//delete files until less than retention size
//find first file that exceeds retention time
}
//delete files living longer than retention limit
//remove file from fileset
return 0;
}
......@@ -124,13 +138,102 @@ void walRemoveAllOldFiles(void *handle) {
}
#endif
static int walRoll(SWal *pWal) {
int code = 0;
code = tfClose(pWal->curIdxTfd);
if(code != 0) {
return code;
}
code = tfClose(pWal->curLogTfd);
if(code != 0) {
return code;
}
int64_t idxTfd, logTfd;
//create new file
int64_t newFileFirstVersion = pWal->lastVersion + 1;
char fnameStr[WAL_FILE_LEN];
sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, newFileFirstVersion);
idxTfd = tfOpenCreateWrite(fnameStr);
sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, newFileFirstVersion);
logTfd = tfOpenCreateWrite(fnameStr);
taosArrayPush(pWal->fileSet, &newFileFirstVersion);
//switch file
pWal->curIdxTfd = idxTfd;
pWal->curLogTfd = logTfd;
//change status
pWal->curFileLastVersion = -1;
pWal->curFileFirstVersion = newFileFirstVersion;
pWal->curVersion = newFileFirstVersion;
pWal->curLogOffset = 0;
pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE;
pWal->lastFileName = newFileFirstVersion;
pWal->lastFileWriteSize = 0;
pWal->lastRollSeq = walGetSeq();
return 0;
}
int walChangeFileToLast(SWal *pWal) {
int64_t idxTfd, logTfd;
int64_t* pRet = taosArrayGetLast(pWal->fileSet);
ASSERT(pRet != NULL);
int64_t fname = *pRet;
char fnameStr[WAL_FILE_LEN];
sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname);
idxTfd = tfOpenReadWrite(fnameStr);
sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname);
logTfd = tfOpenReadWrite(fnameStr);
//switch file
pWal->curIdxTfd = idxTfd;
pWal->curLogTfd = logTfd;
//change status
pWal->curFileLastVersion = -1;
pWal->curFileFirstVersion = fname;
pWal->curVersion = fname;
pWal->curLogOffset = 0;
pWal->curStatus = WAL_CUR_FILE_WRITABLE;
return 0;
}
int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
int code = 0;
//get index file
if(!tfValid(pWal->curIdxTfd)) {
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno));
}
int64_t writeBuf[2] = { ver, offset };
int size = tfWrite(pWal->curIdxTfd, writeBuf, sizeof(writeBuf));
if(size != sizeof(writeBuf)) {
//TODO:
}
return 0;
}
int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t bodyLen) {
if (pWal == NULL) return -1;
// no wal
if (!tfValid(pWal->curLogTfd)) return 0;
if (pWal->level == TAOS_WAL_NOLOG) return 0;
if (index > pWal->lastVersion + 1) return -1;
if (index == pWal->lastVersion + 1) {
int64_t passed = walGetSeq() - pWal->lastRollSeq;
if(passed > pWal->rollPeriod) {
walRoll(pWal);
} else if(pWal->lastFileWriteSize > pWal->segSize) {
walRoll(pWal);
} else {
walChangeFileToLast(pWal);
}
} else {
//reject skip log or rewrite log
//must truncate explicitly first
return -1;
}
if (!tfValid(pWal->curLogTfd)) return 0;
pWal->head.version = index;
int32_t code = 0;
......@@ -155,7 +258,11 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno));
}
//TODO:write idx
walWriteIndex(pWal, index, pWal->curLogOffset);
pWal->curLogOffset += sizeof(SWalHead) + bodyLen;
//set status
pWal->lastVersion = index;
pthread_mutex_unlock(&pWal->mutex);
......
source/util/src/tfile.c
浏览文件 @ d3861996
......@@ -53,6 +53,11 @@ static int64_t tfOpenImp(int32_t fd) {
return rid;
}
int64_t tfOpenRead(const char *pathname, int32_t flags) {
int32_t fd = taosOpenFileRead(pathname);
return tfOpenImp(fd);
}
int64_t tfOpenReadWrite(const char *pathname, int32_t flags) {
int32_t fd = taosOpenFileReadWrite(pathname);
return tfOpenImp(fd);
......
source/util/src/tutil.c
浏览文件 @ d3861996
......@@ -410,3 +410,10 @@ char *taosIpStr(uint32_t ipInt) {
return ipStr;
}
void taosIp2String(uint32_t ip, char *str) {
sprintf(str, "%u.%u.%u.%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24));
}
void taosIpPort2String(uint32_t ip, uint16_t port, char *str) {
sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port);
}
\ No newline at end of file
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或