diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 72ce3a946cae3492012b15fc9c07b50e3382fcf8..8f89df40d0b48cab3e93036ef9ff4f6ce96aaeae 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -990,6 +990,22 @@ typedef struct { /* data */ } SAlterTableRsp; +typedef struct { + /* data */ +} SDropStableReq; + +typedef struct { + /* data */ +} SDropStableRsp; + +typedef struct { + /* data */ +} SUpdateTagValReq; + +typedef struct { + /* data */ +} SUpdateTagValRsp; + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tglobal.h b/include/common/tglobal.h index ffabe0d935622bf6beffe9d94fc8c9e705a8857a..ad2924b1e4979ef6e7c2d1dfb7b80f90c6612551 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -57,6 +57,7 @@ extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; extern char tsTempDir[]; +extern int64_t tsMaxVnodeQueuedBytes; //query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing diff --git a/include/server/mnode/mnode.h b/include/server/mnode/mnode.h index 7ea50a92cefe8869c00c84a85fe7b9c90347c168..bbc0613c2b339fa730aab9cde411c3711f718695 100644 --- a/include/server/mnode/mnode.h +++ b/include/server/mnode/mnode.h @@ -118,7 +118,7 @@ typedef struct { int32_t mnodeGetStatistics(SMnodeStat *stat); /** - * Get the auth information. + * Get the auth information of Mnode. * * @param user, username. * @param spi, security parameter index. diff --git a/include/server/vnode/vnode.h b/include/server/vnode/vnode.h index 8fd4fd433f045042145ccfa2fd8339534aef5275..00decfe338d98d6989a21bef5e404372a4d03c63 100644 --- a/include/server/vnode/vnode.h +++ b/include/server/vnode/vnode.h @@ -49,7 +49,7 @@ typedef struct { } SVnodeFp; typedef struct { - SVnodeFp fp; + SVnodeFp fp; } SVnodePara; /** diff --git a/include/util/tstep.h b/include/util/tstep.h index 87e95edd97e6de0a80dc6ac71678b3c481a76f24..ffc3f6ccf71ee37f00fdc23e394b4a2083aa6603 100644 --- a/include/util/tstep.h +++ b/include/util/tstep.h @@ -20,14 +20,14 @@ extern "C" { #endif -typedef int32_t (*InitFp)(void **obj); -typedef void (*CleanupFp)(void **obj); +typedef int32_t (*InitFp)(); +typedef void (*CleanupFp)(); typedef void (*ReportFp)(char *name, char *desc); struct SSteps *taosStepInit(int32_t maxsize, ReportFp fp); int32_t taosStepExec(struct SSteps *steps); void taosStepCleanup(struct SSteps *steps); -int32_t taosStepAdd(struct SSteps *steps, char *name, void **obj, InitFp initFp, CleanupFp cleanupFp); +int32_t taosStepAdd(struct SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp); #ifdef __cplusplus } diff --git a/include/util/tworker.h b/include/util/tworker.h index 156ced383e0a9d81cbddd22c293853c81a41e88c..367c1a24b98ff86f18af971805f832484973fa28 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -20,14 +20,16 @@ extern "C" { #endif -typedef int32_t (*ProcessReqFp)(void *ahandle, void *msg); -typedef void (*SendRspFp)(void *ahandle, void *msg, int32_t qtype, int32_t code); +typedef int32_t (*ProcessStartFp)(void *ahandle, void *pMsg, int32_t qtype); +typedef void (*ProcessEndFp)(void *ahandle, void *pMsg, int32_t qtype, int32_t code); -struct SWorkerPool; +typedef bool (*ProcessWriteStartFp)(void *ahandle, void *pMsg, int32_t qtype); +typedef void (*ProcessWriteSyncFp)(void *ahandle, int32_t code); +typedef void (*ProcessWriteEndFp)(void *ahandle, void *pMsg, int32_t qtype); -typedef struct { - pthread_t thread; // thread +typedef struct SWorker { int32_t id; // worker ID + pthread_t thread; // thread struct SWorkerPool *pool; } SWorker; @@ -35,18 +37,42 @@ typedef struct SWorkerPool { int32_t max; // max number of workers int32_t min; // min number of workers int32_t num; // current number of workers - void * qset; + taos_qset qset; const char * name; + ProcessStartFp startFp; + ProcessEndFp endFp; SWorker * workers; - ProcessReqFp reqFp; - SendRspFp rspFp; pthread_mutex_t mutex; } SWorkerPool; -int32_t tWorkerInit(SWorkerPool *pPool); -void tWorkerCleanup(SWorkerPool *pPool); -void * tWorkerAllocQueue(SWorkerPool *pPool, void *ahandle); -void tWorkerFreeQueue(SWorkerPool *pPool, void *pQueue); +typedef struct SWriteWorker { + int32_t id; // worker id + pthread_t thread; // thread + taos_qall qall; + taos_qset qset; // queue set + struct SWriteWorkerPool *pool; +} SWriteWorker; + +typedef struct SWriteWorkerPool { + int32_t max; // max number of workers + int32_t nextId; // from 0 to max-1, cyclic + const char * name; + ProcessWriteStartFp startFp; + ProcessWriteSyncFp syncFp; + ProcessWriteEndFp endFp; + SWriteWorker * workers; + pthread_mutex_t mutex; +} SWriteWorkerPool; + +int32_t tWorkerInit(SWorkerPool *pool); +void tWorkerCleanup(SWorkerPool *pool); +taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle); +void tWorkerFreeQueue(SWorkerPool *pool, taos_queue queue); + +int32_t tWriteWorkerInit(SWriteWorkerPool *pool); +void tWriteWorkerCleanup(SWriteWorkerPool *pool); +taos_queue tWriteWorkerAllocQueue(SWriteWorkerPool *pool, void *ahandle); +void tWriteWorkerFreeQueue(SWriteWorkerPool *pool, taos_queue queue); #ifdef __cplusplus } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9eab95cd41243d62fc9a51dbc402d4b08efdcdc9..745e3ad1eeabc313f42d52800667226bcc1285c5 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -60,6 +60,7 @@ float tsRatioOfQueryCores = 1.0f; int8_t tsDaylight = 0; int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; +int64_t tsMaxVnodeQueuedBytes = 1024*1024*1024; //1GB /* * denote if the server needs to compress response message at the application layer to client, including query rsp, diff --git a/source/server/dnode/inc/dnodeCfg.h b/source/server/dnode/inc/dnodeCfg.h index 1565e7649ac6b1c5be7fc52719323a048293e217..eda6231579fd35611bc59f39e48e9ea354915eab 100644 --- a/source/server/dnode/inc/dnodeCfg.h +++ b/source/server/dnode/inc/dnodeCfg.h @@ -21,21 +21,14 @@ extern "C" { #endif #include "dnodeInt.h" -typedef struct SDnCfg { - int32_t dnodeId; - int32_t dropped; - char clusterId[TSDB_CLUSTER_ID_LEN]; - char file[PATH_MAX + 20]; - pthread_mutex_t mutex; -} SDnCfg; -int32_t dnodeInitCfg(SDnCfg **cfg); -void dnodeCleanupCfg(SDnCfg **cfg); -void dnodeUpdateCfg(SDnCfg *cfg, SDnodeCfg *data); -int32_t dnodeGetDnodeId(SDnCfg *cfg); -void dnodeGetClusterId(SDnCfg *cfg, char *clusterId); -void dnodeGetCfg(SDnCfg *cfg, int32_t *dnodeId, char *clusterId); -void dnodeSetDropped(SDnCfg *cfg); +int32_t dnodeInitCfg(); +void dnodeCleanupCfg(); +void dnodeUpdateCfg(SDnodeCfg *data); +int32_t dnodeGetDnodeId(); +void dnodeGetClusterId(char *clusterId); +void dnodeGetCfg(int32_t *dnodeId, char *clusterId); +void dnodeSetDropped(); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeCheck.h b/source/server/dnode/inc/dnodeCheck.h index 29172ba4dfc7a19eded484e8cfc45982f562d172..b6fbf1eabd5f2322c0b31192c2b11c7b976f28fc 100644 --- a/source/server/dnode/inc/dnodeCheck.h +++ b/source/server/dnode/inc/dnodeCheck.h @@ -21,11 +21,9 @@ extern "C" { #endif #include "dnodeInt.h" -typedef struct SDnCheck { -} SDnCheck; -int32_t dnodeInitCheck(SDnCheck **check); -void dnodeCleanupCheck(SDnCheck **check); +int32_t dnodeInitCheck(); +void dnodeCleanupCheck(); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeEps.h b/source/server/dnode/inc/dnodeEps.h index 8019a819337caa4989fb5559e08f561fa4d08b9f..4f25884021648377392016fe7daced6e9b0b4a00 100644 --- a/source/server/dnode/inc/dnodeEps.h +++ b/source/server/dnode/inc/dnodeEps.h @@ -19,22 +19,12 @@ #ifdef __cplusplus extern "C" { #endif -#include "thash.h" #include "dnodeInt.h" -typedef struct SDnEps { - int32_t dnodeId; - int32_t dnodeNum; - SDnodeEp * dnodeList; - SHashObj * dnodeHash; - char file[PATH_MAX + 20]; - pthread_mutex_t mutex; -} SDnEps; - -int32_t dnodeInitEps(SDnEps **eps); -void dnodeCleanupEps(SDnEps **eps); -void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data); -bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr); +int32_t dnodeInitEps(); +void dnodeCleanupEps(); +void dnodeUpdateEps(SDnodeEps *data); +bool dnodeIsDnodeEpChanged(int32_t dnodeId, char *epstr); void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); #ifdef __cplusplus diff --git a/source/server/dnode/inc/dnodeInt.h b/source/server/dnode/inc/dnodeInt.h index 9b56147ca0d94bbeca0c93b3b593e479ee50b6c4..82cdfb52bf778e5f591d9a7b5c101b5de75169b1 100644 --- a/source/server/dnode/inc/dnodeInt.h +++ b/source/server/dnode/inc/dnodeInt.h @@ -19,36 +19,12 @@ #ifdef __cplusplus extern "C" { #endif -#include "taoserror.h" #include "taosmsg.h" -#include "tglobal.h" #include "tlog.h" #include "trpc.h" -#include "tstep.h" #include "dnode.h" -struct SDnCfg; -struct SDnCheck; -struct SDnEps; -struct SDnMnEps; -struct SDnStatus; -struct SDnTelem; -struct SDnTrans; -struct SDnMain; - -typedef struct SDnode { - struct SSteps* steps; - struct SDnCfg* cfg; - struct SDnCheck* check; - struct SDnEps* eps; - struct SDnMnEps* meps; - struct SDnStatus* status; - struct SDnTelem* telem; - struct SDnTrans* trans; - struct SDnMain* main; -} SDnode; - -SDnode* dnodeInst(); +extern int32_t dDebugFlag; #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} diff --git a/source/server/dnode/inc/dnodeMain.h b/source/server/dnode/inc/dnodeMain.h index 08f6e108307306b6473548c22ceb8c74e576f896..245ede000175119169c46735fa7717128cacdc6f 100644 --- a/source/server/dnode/inc/dnodeMain.h +++ b/source/server/dnode/inc/dnodeMain.h @@ -27,14 +27,8 @@ typedef enum { TD_RUN_STAT_STOPPED } RunStat; -typedef struct SDnMain { - RunStat runStatus; - void * dnodeTimer; - SStartupStep startup; -} SDnMain; - -int32_t dnodeInitMain(SDnMain **main); -void dnodeCleanupMain(SDnMain **main); +int32_t dnodeInitMain(); +void dnodeCleanupMain(); int32_t dnodeInitStorage(); void dnodeCleanupStorage(); void dnodeReportStartup(char *name, char *desc); @@ -42,6 +36,9 @@ void dnodeReportStartupFinished(char *name, char *desc); void dnodeProcessStartupReq(SRpcMsg *pMsg); void dnodeProcessCreateMnodeReq(SRpcMsg *pMsg); void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg); +RunStat dnodeGetRunStat(); +void dnodeSetRunStat(); +void* dnodeGetTimer(); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeMnodeEps.h b/source/server/dnode/inc/dnodeMnodeEps.h index a70c621046ce2e4c34032fa6a1a21455a6d25539..c890f6921d3e1e75aaad9054dffa08bcfb1fc520 100644 --- a/source/server/dnode/inc/dnodeMnodeEps.h +++ b/source/server/dnode/inc/dnodeMnodeEps.h @@ -21,19 +21,12 @@ extern "C" { #endif #include "dnodeInt.h" -typedef struct SDnMnEps { - SRpcEpSet mnodeEpSet; - SMInfos mnodeInfos; - char file[PATH_MAX + 20]; - pthread_mutex_t mutex; -} SDnMnEps; - -int32_t dnodeInitMnodeEps(SDnMnEps **meps); -void dnodeCleanupMnodeEps(SDnMnEps **meps); -void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *pMinfos); -void dnodeUpdateMnodeFromPeer(SDnMnEps *meps, SRpcEpSet *pEpSet); -void dnodeGetEpSetForPeer(SDnMnEps *meps, SRpcEpSet *epSet); -void dnodeGetEpSetForShell(SDnMnEps *meps, SRpcEpSet *epSet); +int32_t dnodeInitMnodeEps(); +void dnodeCleanupMnodeEps(); +void dnodeUpdateMnodeFromStatus(SMInfos *pMinfos); +void dnodeUpdateMnodeFromPeer(SRpcEpSet *pEpSet); +void dnodeGetEpSetForPeer(SRpcEpSet *epSet); +void dnodeGetEpSetForShell(SRpcEpSet *epSet); void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); #ifdef __cplusplus diff --git a/source/server/dnode/inc/dnodeStatus.h b/source/server/dnode/inc/dnodeStatus.h index 65a349ba6d03e83f49d8cd5b225477a1124c996e..f0473b93f1be274a03ab7a8e513ef45738fe9e29 100644 --- a/source/server/dnode/inc/dnodeStatus.h +++ b/source/server/dnode/inc/dnodeStatus.h @@ -21,14 +21,8 @@ extern "C" { #endif #include "dnodeInt.h" -typedef struct SDnStatus { - void * dnodeTimer; - void * statusTimer; - uint32_t rebootTime; -} SDnStatus; - -int32_t dnodeInitStatus(SDnStatus **status); -void dnodeCleanupStatus(SDnStatus **status); +int32_t dnodeInitStatus(); +void dnodeCleanupStatus(); void dnodeProcessStatusRsp(SRpcMsg *pMsg); #ifdef __cplusplus diff --git a/source/server/dnode/inc/dnodeTelem.h b/source/server/dnode/inc/dnodeTelem.h index 94356fffba9319c1df8cf22dc545d4ecee2e7626..4945879e64174065882841a2d366f4570b3d7070 100644 --- a/source/server/dnode/inc/dnodeTelem.h +++ b/source/server/dnode/inc/dnodeTelem.h @@ -21,21 +21,8 @@ extern "C" { #endif #include "dnodeInt.h" -/* - * sem_timedwait is NOT implemented on MacOSX - * thus we use pthread_mutex_t/pthread_cond_t to simulate - */ -typedef struct SDnTelem { - bool enable; - pthread_mutex_t lock; - pthread_cond_t cond; - volatile int32_t exit; - pthread_t thread; - char email[TSDB_FQDN_LEN]; -} SDnTelem; - -int32_t dnodeInitTelem(SDnTelem **telem); -void dnodeCleanupTelem(SDnTelem **telem); +int32_t dnodeInitTelem(); +void dnodeCleanupTelem(); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeTrans.h b/source/server/dnode/inc/dnodeTrans.h index d9016f0c7bc689262de8d83644044a27b4f4cc98..631c69d11c66246abff080d03456b857aa2d94ac 100644 --- a/source/server/dnode/inc/dnodeTrans.h +++ b/source/server/dnode/inc/dnodeTrans.h @@ -21,20 +21,8 @@ extern "C" { #endif #include "dnodeInt.h" -typedef void (*RpcMsgFp)( SRpcMsg *pMsg); - -typedef struct SDnTrans { - void * serverRpc; - void * clientRpc; - void * shellRpc; - int32_t queryReqNum; - int32_t submitReqNum; - RpcMsgFp peerMsgFp[TSDB_MSG_TYPE_MAX]; - RpcMsgFp shellMsgFp[TSDB_MSG_TYPE_MAX]; -} SDnTrans; - -int32_t dnodeInitTrans(SDnTrans **rans); -void dnodeCleanupTrans(SDnTrans **trans); +int32_t dnodeInitTrans(); +void dnodeCleanupTrans(); void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); diff --git a/source/server/dnode/src/dnodeCfg.c b/source/server/dnode/src/dnodeCfg.c index ee8a17db082de9ada1da9028feb562dc0ebb32c1..f9ed4914645af37ef5c7d8184f25c0d8499328cc 100644 --- a/source/server/dnode/src/dnodeCfg.c +++ b/source/server/dnode/src/dnodeCfg.c @@ -16,56 +16,65 @@ #define _DEFAULT_SOURCE #include "os.h" #include "cJSON.h" +#include "tglobal.h" #include "dnodeCfg.h" -static int32_t dnodeReadCfg(SDnCfg *cfg) { +static struct DnCfg { + int32_t dnodeId; + int32_t dropped; + char clusterId[TSDB_CLUSTER_ID_LEN]; + char file[PATH_MAX + 20]; + pthread_mutex_t mutex; +} tsDcfg; + +static int32_t dnodeReadCfg() { int32_t len = 0; int32_t maxLen = 200; char * content = calloc(1, maxLen + 1); cJSON * root = NULL; FILE * fp = NULL; - fp = fopen(cfg->file, "r"); + fp = fopen(tsDcfg.file, "r"); if (!fp) { - dDebug("file %s not exist", cfg->file); + dDebug("file %s not exist", tsDcfg.file); goto PARSE_CFG_OVER; } len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { - dError("failed to read %s since content is null", cfg->file); + dError("failed to read %s since content is null", tsDcfg.file); goto PARSE_CFG_OVER; } content[len] = 0; root = cJSON_Parse(content); if (root == NULL) { - dError("failed to read %s since invalid json format", cfg->file); + dError("failed to read %s since invalid json format", tsDcfg.file); goto PARSE_CFG_OVER; } cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", cfg->file); + dError("failed to read %s since dnodeId not found", tsDcfg.file); goto PARSE_CFG_OVER; } - cfg->dnodeId = (int32_t)dnodeId->valueint; + tsDcfg.dnodeId = (int32_t)dnodeId->valueint; cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", cfg->file); + dError("failed to read %s since dropped not found", tsDcfg.file); goto PARSE_CFG_OVER; } - cfg->dropped = (int32_t)dropped->valueint; + tsDcfg.dropped = (int32_t)dropped->valueint; cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read %s since clusterId not found", cfg->file); + dError("failed to read %s since clusterId not found", tsDcfg.file); goto PARSE_CFG_OVER; } - tstrncpy(cfg->clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN); + tstrncpy(tsDcfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN); - dInfo("successed to read %s", cfg->file); + dInfo("successed to read %s", tsDcfg.file); PARSE_CFG_OVER: if (content != NULL) free(content); @@ -76,10 +85,10 @@ PARSE_CFG_OVER: return 0; } -static int32_t dnodeWriteCfg(SDnCfg *cfg) { - FILE *fp = fopen(cfg->file, "w"); +static int32_t dnodeWriteCfg() { + FILE *fp = fopen(tsDcfg.file, "w"); if (!fp) { - dError("failed to write %s since %s", cfg->file, strerror(errno)); + dError("failed to write %s since %s", tsDcfg.file, strerror(errno)); return -1; } @@ -88,9 +97,9 @@ static int32_t dnodeWriteCfg(SDnCfg *cfg) { char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", cfg->dnodeId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", cfg->dropped); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", cfg->clusterId); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsDcfg.dnodeId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", tsDcfg.dropped); + len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsDcfg.clusterId); len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); @@ -99,27 +108,23 @@ static int32_t dnodeWriteCfg(SDnCfg *cfg) { free(content); terrno = 0; - dInfo("successed to write %s", cfg->file); + dInfo("successed to write %s", tsDcfg.file); return 0; } -int32_t dnodeInitCfg(SDnCfg **out) { - SDnCfg* cfg = calloc(1, sizeof(SDnCfg)); - if (cfg == NULL) return -1; - - cfg->dnodeId = 0; - cfg->dropped = 0; - cfg->clusterId[0] = 0; - snprintf(cfg->file, sizeof(cfg->file), "%s/dnodeCfg.json", tsDnodeDir); - pthread_mutex_init(&cfg->mutex, NULL); - *out = cfg; - - int32_t ret = dnodeReadCfg(cfg); +int32_t dnodeInitCfg() { + tsDcfg.dnodeId = 0; + tsDcfg.dropped = 0; + tsDcfg.clusterId[0] = 0; + snprintf(tsDcfg.file, sizeof(tsDcfg.file), "%s/dnodeCfg.json", tsDnodeDir); + pthread_mutex_init(&tsDcfg.mutex, NULL); + + int32_t ret = dnodeReadCfg(); if (ret == 0) { dInfo("dnode cfg is initialized"); } - if (cfg->dropped) { + if (tsDcfg.dropped) { dInfo("dnode is dropped and start to exit"); return -1; } @@ -127,51 +132,47 @@ int32_t dnodeInitCfg(SDnCfg **out) { return ret; } -void dnodeCleanupCfg(SDnCfg **out) { - SDnCfg* cfg = *out; - *out = NULL; - - pthread_mutex_destroy(&cfg->mutex); - free(cfg); +void dnodeCleanupCfg() { + pthread_mutex_destroy(&tsDcfg.mutex); } -void dnodeUpdateCfg(SDnCfg *cfg, SDnodeCfg *data) { - if (cfg == NULL || cfg->dnodeId == 0) return; +void dnodeUpdateCfg(SDnodeCfg *data) { + if (tsDcfg.dnodeId != 0) return; - pthread_mutex_lock(&cfg->mutex); + pthread_mutex_lock(&tsDcfg.mutex); - cfg->dnodeId = data->dnodeId; - tstrncpy(cfg->clusterId, data->clusterId, TSDB_CLUSTER_ID_LEN); - dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId); + tsDcfg.dnodeId = data->dnodeId; + tstrncpy(tsDcfg.clusterId, data->clusterId, TSDB_CLUSTER_ID_LEN); + dInfo("dnodeId is set to %d, clusterId is set to %s", data->dnodeId, data->clusterId); - dnodeWriteCfg(cfg); - pthread_mutex_unlock(&cfg->mutex); + dnodeWriteCfg(); + pthread_mutex_unlock(&tsDcfg.mutex); } -void dnodeSetDropped(SDnCfg *cfg) { - pthread_mutex_lock(&cfg->mutex); - cfg->dropped = 1; - dnodeWriteCfg(cfg); - pthread_mutex_unlock(&cfg->mutex); +void dnodeSetDropped() { + pthread_mutex_lock(&tsDcfg.mutex); + tsDcfg.dropped = 1; + dnodeWriteCfg(); + pthread_mutex_unlock(&tsDcfg.mutex); } -int32_t dnodeGetDnodeId(SDnCfg *cfg) { +int32_t dnodeGetDnodeId() { int32_t dnodeId = 0; - pthread_mutex_lock(&cfg->mutex); - dnodeId = cfg->dnodeId; - pthread_mutex_unlock(&cfg->mutex); + pthread_mutex_lock(&tsDcfg.mutex); + dnodeId = tsDcfg.dnodeId; + pthread_mutex_unlock(&tsDcfg.mutex); return dnodeId; } -void dnodeGetClusterId(SDnCfg *cfg, char *clusterId) { - pthread_mutex_lock(&cfg->mutex); - tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); - pthread_mutex_unlock(&cfg->mutex); +void dnodeGetClusterId(char *clusterId) { + pthread_mutex_lock(&tsDcfg.mutex); + tstrncpy(clusterId, tsDcfg.clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&tsDcfg.mutex); } -void dnodeGetCfg(SDnCfg *cfg, int32_t *dnodeId, char *clusterId) { - pthread_mutex_lock(&cfg->mutex); - *dnodeId = cfg->dnodeId; - tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); - pthread_mutex_unlock(&cfg->mutex); +void dnodeGetCfg(int32_t *dnodeId, char *clusterId) { + pthread_mutex_lock(&tsDcfg.mutex); + *dnodeId = tsDcfg.dnodeId; + tstrncpy(clusterId, tsDcfg.clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&tsDcfg.mutex); } diff --git a/source/server/dnode/src/dnodeCheck.c b/source/server/dnode/src/dnodeCheck.c index 8f561b1ea05174c11d3ba79970b098cf36325a22..b59e4bd4e027fae69f514c8d43c5f6d958932bfb 100644 --- a/source/server/dnode/src/dnodeCheck.c +++ b/source/server/dnode/src/dnodeCheck.c @@ -118,7 +118,7 @@ static int32_t dnodeCheckMem() { } static int32_t dnodeCheckDisk() { -#if 0 +#if 0 taosGetDisk(); if (tsAvailDataDirGB < tsMinimalDataDirGB) { @@ -145,12 +145,7 @@ static int32_t dnodeCheckAccess() { return 0; } static int32_t dnodeCheckVersion() { return 0; } static int32_t dnodeCheckDatafile() { return 0; } -int32_t dnodeInitCheck(SDnCheck **out) { - SDnCheck *check = calloc(1, sizeof(SDnCheck)); - if (check == NULL) return -1; - - *out = check; - +int32_t dnodeInitCheck() { if (dnodeCheckNetwork() != 0) { dError("failed to check network"); return -1; @@ -195,9 +190,4 @@ int32_t dnodeInitCheck(SDnCheck **out) { return 0; } -void dnodeCleanupCheck(SDnCheck **out) { - SDnCheck *check = *out; - *out = NULL; - - free(check); -} \ No newline at end of file +void dnodeCleanupCheck() {} \ No newline at end of file diff --git a/source/server/dnode/src/dnodeEps.c b/source/server/dnode/src/dnodeEps.c index 317a9968fa4155cea5aa821a7b459e3621543d54..d5bb77bde674325d59ba4443b66c457aefb8493b 100644 --- a/source/server/dnode/src/dnodeEps.c +++ b/source/server/dnode/src/dnodeEps.c @@ -16,86 +16,96 @@ #define _DEFAULT_SOURCE #include "os.h" #include "cJSON.h" +#include "thash.h" #include "tglobal.h" #include "dnodeEps.h" #include "dnodeCfg.h" -static void dnodePrintEps(SDnEps *eps) { - dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); - for (int32_t i = 0; i < eps->dnodeNum; i++) { - SDnodeEp *ep = &eps->dnodeList[i]; +static struct { + int32_t dnodeId; + int32_t dnodeNum; + SDnodeEp * dnodeList; + SHashObj * dnodeHash; + char file[PATH_MAX + 20]; + pthread_mutex_t mutex; +} tsDeps; + +static void dnodePrintEps() { + dDebug("print dnodeEp, dnodeNum:%d", tsDeps.dnodeNum); + for (int32_t i = 0; i < tsDeps.dnodeNum; i++) { + SDnodeEp *ep = &tsDeps.dnodeList[i]; dDebug("dnode:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); } } -static void dnodeResetEps(SDnEps *eps, SDnodeEps *data) { +static void dnodeResetEps(SDnodeEps *data) { assert(data != NULL); - if (data->dnodeNum > eps->dnodeNum) { + if (data->dnodeNum > tsDeps.dnodeNum) { SDnodeEp *tmp = calloc(data->dnodeNum, sizeof(SDnodeEp)); if (tmp == NULL) return; - tfree(eps->dnodeList); - eps->dnodeList = tmp; - eps->dnodeNum = data->dnodeNum; - memcpy(eps->dnodeList, data->dnodeEps, eps->dnodeNum * sizeof(SDnodeEp)); - dnodePrintEps(eps); + tfree(tsDeps.dnodeList); + tsDeps.dnodeList = tmp; + tsDeps.dnodeNum = data->dnodeNum; + memcpy(tsDeps.dnodeList, data->dnodeEps, tsDeps.dnodeNum * sizeof(SDnodeEp)); + dnodePrintEps(); - for (int32_t i = 0; i < eps->dnodeNum; ++i) { - SDnodeEp *ep = &eps->dnodeList[i]; - taosHashPut(eps->dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp)); + for (int32_t i = 0; i < tsDeps.dnodeNum; ++i) { + SDnodeEp *ep = &tsDeps.dnodeList[i]; + taosHashPut(tsDeps.dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp)); } } } -static int32_t dnodeReadEps(SDnEps *eps) { +static int32_t dnodeReadEps() { int32_t len = 0; int32_t maxLen = 30000; char * content = calloc(1, maxLen + 1); cJSON * root = NULL; FILE * fp = NULL; - fp = fopen(eps->file, "r"); + fp = fopen(tsDeps.file, "r"); if (!fp) { - dDebug("file %s not exist", eps->file); + dDebug("file %s not exist", tsDeps.file); goto PRASE_EPS_OVER; } len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { - dError("failed to read %s since content is null", eps->file); + dError("failed to read %s since content is null", tsDeps.file); goto PRASE_EPS_OVER; } content[len] = 0; root = cJSON_Parse(content); if (root == NULL) { - dError("failed to read %s since invalid json format", eps->file); + dError("failed to read %s since invalid json format", tsDeps.file); goto PRASE_EPS_OVER; } cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum"); if (!dnodeNum || dnodeNum->type != cJSON_Number) { - dError("failed to read %s since dnodeNum not found", eps->file); + dError("failed to read %s since dnodeNum not found", tsDeps.file); goto PRASE_EPS_OVER; } cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos"); if (!dnodeInfos || dnodeInfos->type != cJSON_Array) { - dError("failed to read %s since dnodeInfos not found", eps->file); + dError("failed to read %s since dnodeInfos not found", tsDeps.file); goto PRASE_EPS_OVER; } int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos); if (dnodeInfosSize != dnodeNum->valueint) { - dError("failed to read %s since dnodeInfos size:%d not matched dnodeNum:%d", eps->file, dnodeInfosSize, + dError("failed to read %s since dnodeInfos size:%d not matched dnodeNum:%d", tsDeps.file, dnodeInfosSize, (int32_t)dnodeNum->valueint); goto PRASE_EPS_OVER; } - eps->dnodeNum = dnodeInfosSize; - eps->dnodeList = calloc(dnodeInfosSize, sizeof(SDnodeEp)); - if (eps->dnodeList == NULL) { + tsDeps.dnodeNum = dnodeInfosSize; + tsDeps.dnodeList = calloc(dnodeInfosSize, sizeof(SDnodeEp)); + if (tsDeps.dnodeList == NULL) { dError("failed to calloc dnodeEpList since %s", strerror(errno)); goto PRASE_EPS_OVER; } @@ -104,40 +114,40 @@ static int32_t dnodeReadEps(SDnEps *eps) { cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i); if (dnodeInfo == NULL) break; - SDnodeEp *ep = &eps->dnodeList[i]; + SDnodeEp *ep = &tsDeps.dnodeList[i]; cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId"); if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s, dnodeId not found", eps->file); + dError("failed to read %s, dnodeId not found", tsDeps.file); goto PRASE_EPS_OVER; } ep->dnodeId = (int32_t)dnodeId->valueint; cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { - dError("failed to read %s, dnodeFqdn not found", eps->file); + dError("failed to read %s, dnodeFqdn not found", tsDeps.file); goto PRASE_EPS_OVER; } tstrncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort"); if (!dnodePort || dnodePort->type != cJSON_Number) { - dError("failed to read %s, dnodePort not found", eps->file); + dError("failed to read %s, dnodePort not found", tsDeps.file); goto PRASE_EPS_OVER; } ep->dnodePort = (uint16_t)dnodePort->valueint; } - dInfo("succcessed to read file %s", eps->file); - dnodePrintEps(eps); + dInfo("succcessed to read file %s", tsDeps.file); + dnodePrintEps(); PRASE_EPS_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); if (fp != NULL) fclose(fp); - if (dnodeIsDnodeEpChanged(eps, eps->dnodeId, tsLocalEp)) { - dError("dnode:%d, localEp different from %s dnodeEps.json and need reconfigured", eps->dnodeId, tsLocalEp); + if (dnodeIsDnodeEpChanged(tsDeps.dnodeId, tsLocalEp)) { + dError("dnode:%d, localEp different from %s dnodeEps.json and need reconfigured", tsDeps.dnodeId, tsLocalEp); return -1; } @@ -145,10 +155,10 @@ PRASE_EPS_OVER: return 0; } -static int32_t dnodeWriteEps(SDnEps *eps) { - FILE *fp = fopen(eps->file, "w"); +static int32_t dnodeWriteEps() { + FILE *fp = fopen(tsDeps.file, "w"); if (!fp) { - dError("failed to write %s since %s", eps->file, strerror(errno)); + dError("failed to write %s since %s", tsDeps.file, strerror(errno)); return -1; } @@ -157,14 +167,14 @@ static int32_t dnodeWriteEps(SDnEps *eps) { char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeNum\": %d,\n", eps->dnodeNum); + len += snprintf(content + len, maxLen - len, " \"dnodeNum\": %d,\n", tsDeps.dnodeNum); len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n"); - for (int32_t i = 0; i < eps->dnodeNum; ++i) { - SDnodeEp *ep = &eps->dnodeList[i]; + for (int32_t i = 0; i < tsDeps.dnodeNum; ++i) { + SDnodeEp *ep = &tsDeps.dnodeList[i]; len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", ep->dnodeId); len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn); len += snprintf(content + len, maxLen - len, " \"dnodePort\": %u\n", ep->dnodePort); - if (i < eps->dnodeNum - 1) { + if (i < tsDeps.dnodeNum - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { len += snprintf(content + len, maxLen - len, " }]\n"); @@ -178,24 +188,20 @@ static int32_t dnodeWriteEps(SDnEps *eps) { free(content); terrno = 0; - dInfo("successed to write %s", eps->file); + dInfo("successed to write %s", tsDeps.file); return 0; } -int32_t dnodeInitEps(SDnEps **out) { - SDnEps *eps = calloc(1, sizeof(SDnEps)); - if (eps == NULL) return -1; +int32_t dnodeInitEps() { + tsDeps.dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (tsDeps.dnodeHash == NULL) return -1; - eps->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (eps->dnodeHash == NULL) return -1; + tsDeps.dnodeId = dnodeGetDnodeId(); + tsDeps.dnodeNum = 0; + snprintf(tsDeps.file, sizeof(tsDeps.file), "%s/dnodeEps.json", tsDnodeDir); + pthread_mutex_init(&tsDeps.mutex, NULL); - eps->dnodeId = dnodeInst()->cfg->dnodeId; - eps->dnodeNum = 0; - snprintf(eps->file, sizeof(eps->file), "%s/dnodeEps.json", tsDnodeDir); - pthread_mutex_init(&eps->mutex, NULL); - *out = eps; - - int32_t ret = dnodeReadEps(eps); + int32_t ret = dnodeReadEps(); if (ret == 0) { dInfo("dnode eps is initialized"); } @@ -203,29 +209,25 @@ int32_t dnodeInitEps(SDnEps **out) { return ret; } -void dnodeCleanupEps(SDnEps **out) { - SDnEps *eps = *out; - *out = NULL; - - pthread_mutex_lock(&eps->mutex); +void dnodeCleanupEps() { + pthread_mutex_lock(&tsDeps.mutex); - if (eps->dnodeList != NULL) { - free(eps->dnodeList); - eps->dnodeList = NULL; + if (tsDeps.dnodeList != NULL) { + free(tsDeps.dnodeList); + tsDeps.dnodeList = NULL; } - if (eps->dnodeHash) { - taosHashCleanup(eps->dnodeHash); - eps->dnodeHash = NULL; + if (tsDeps.dnodeHash) { + taosHashCleanup(tsDeps.dnodeHash); + tsDeps.dnodeHash = NULL; } - eps->dnodeNum = 0; - pthread_mutex_unlock(&eps->mutex); - pthread_mutex_destroy(&eps->mutex); - free(eps); + tsDeps.dnodeNum = 0; + pthread_mutex_unlock(&tsDeps.mutex); + pthread_mutex_destroy(&tsDeps.mutex); } -void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data) { +void dnodeUpdateEps(SDnodeEps *data) { if (data == NULL || data->dnodeNum <= 0) return; data->dnodeNum = htonl(data->dnodeNum); @@ -234,28 +236,28 @@ void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data) { data->dnodeEps[i].dnodePort = htons(data->dnodeEps[i].dnodePort); } - pthread_mutex_lock(&eps->mutex); + pthread_mutex_lock(&tsDeps.mutex); - if (data->dnodeNum != eps->dnodeNum) { - dnodeResetEps(eps, data); - dnodeWriteEps(eps); + if (data->dnodeNum != tsDeps.dnodeNum) { + dnodeResetEps(data); + dnodeWriteEps(); } else { int32_t size = data->dnodeNum * sizeof(SDnodeEp); - if (memcmp(eps->dnodeList, data->dnodeEps, size) != 0) { - dnodeResetEps(eps, data); - dnodeWriteEps(eps); + if (memcmp(tsDeps.dnodeList, data->dnodeEps, size) != 0) { + dnodeResetEps(data); + dnodeWriteEps(); } } - pthread_mutex_unlock(&eps->mutex); + pthread_mutex_unlock(&tsDeps.mutex); } -bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr) { +bool dnodeIsDnodeEpChanged(int32_t dnodeId, char *epstr) { bool changed = false; - pthread_mutex_lock(&eps->mutex); + pthread_mutex_lock(&tsDeps.mutex); - SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t)); + SDnodeEp *ep = taosHashGet(tsDeps.dnodeHash, &dnodeId, sizeof(int32_t)); if (ep != NULL) { char epSaved[TSDB_EP_LEN + 1]; snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); @@ -263,21 +265,20 @@ bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr) { tstrncpy(epstr, epSaved, TSDB_EP_LEN); } - pthread_mutex_unlock(&eps->mutex); + pthread_mutex_unlock(&tsDeps.mutex); return changed; } void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) { - SDnEps *eps = dnodeInst()->eps; - pthread_mutex_lock(&eps->mutex); + pthread_mutex_lock(&tsDeps.mutex); - SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t)); + SDnodeEp *ep = taosHashGet(tsDeps.dnodeHash, &dnodeId, sizeof(int32_t)); if (ep != NULL) { if (port) *port = ep->dnodePort; if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN); if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); } - pthread_mutex_unlock(&eps->mutex); + pthread_mutex_unlock(&tsDeps.mutex); } diff --git a/source/server/dnode/src/dnodeInt.c b/source/server/dnode/src/dnodeInt.c index abb552cb02b9d629de5a14647165eedf6c5b7605..84a7e2565b3a636c9ae5f2d89c5b292137129940 100644 --- a/source/server/dnode/src/dnodeInt.c +++ b/source/server/dnode/src/dnodeInt.c @@ -33,10 +33,7 @@ #include "mnode.h" #include "vnode.h" -SDnode *dnodeInst() { - static SDnode inst = {0}; - return &inst; -} +static struct SSteps *tsSteps; static int32_t dnodeInitVnodeModule(void **unused) { SVnodePara para; @@ -48,58 +45,50 @@ static int32_t dnodeInitVnodeModule(void **unused) { } static int32_t dnodeInitMnodeModule(void **unused) { - SDnode *dnode = dnodeInst(); - SMnodePara para; para.fp.GetDnodeEp = dnodeGetDnodeEp; para.fp.SendMsgToDnode = dnodeSendMsgToDnode; para.fp.SendMsgToMnode = dnodeSendMsgToMnode; para.fp.SendRedirectMsg = dnodeSendRedirectMsg; - para.dnodeId = dnode->cfg->dnodeId; - strncpy(para.clusterId, dnode->cfg->clusterId, sizeof(para.clusterId)); + dnodeGetCfg(¶.dnodeId, para.clusterId); return mnodeInit(para); } int32_t dnodeInit() { - struct SSteps *steps = taosStepInit(24, dnodeReportStartup); - if (steps == NULL) return -1; + tsSteps = taosStepInit(24, dnodeReportStartup); + if (tsSteps == NULL) return -1; - SDnode *dnode = dnodeInst(); + taosStepAdd(tsSteps, "dnode-main", dnodeInitMain, dnodeCleanupMain); + taosStepAdd(tsSteps, "dnode-storage", dnodeInitStorage, dnodeCleanupStorage); + //taosStepAdd(tsSteps, "dnode-tfs", tfInit, tfCleanup); + taosStepAdd(tsSteps, "dnode-rpc", rpcInit, rpcCleanup); + taosStepAdd(tsSteps, "dnode-check", dnodeInitCheck, dnodeCleanupCheck); + taosStepAdd(tsSteps, "dnode-cfg", dnodeInitCfg, dnodeCleanupCfg); + taosStepAdd(tsSteps, "dnode-deps", dnodeInitEps, dnodeCleanupEps); + taosStepAdd(tsSteps, "dnode-meps", dnodeInitMnodeEps, dnodeCleanupMnodeEps); + //taosStepAdd(tsSteps, "dnode-wal", walInit, walCleanUp); + //taosStepAdd(tsSteps, "dnode-sync", syncInit, syncCleanUp); + taosStepAdd(tsSteps, "dnode-vnode", dnodeInitVnodeModule, vnodeCleanup); + taosStepAdd(tsSteps, "dnode-mnode", dnodeInitMnodeModule, mnodeCleanup); + taosStepAdd(tsSteps, "dnode-trans", dnodeInitTrans, dnodeCleanupTrans); + taosStepAdd(tsSteps, "dnode-status", dnodeInitStatus, dnodeCleanupStatus); + taosStepAdd(tsSteps, "dnode-telem", dnodeInitTelem, dnodeCleanupTelem); + //taosStepAdd(tsSteps, "dnode-script",scriptEnvPoolInit, scriptEnvPoolCleanup); - taosStepAdd(steps, "dnode-main", (void **)&dnode->main, (InitFp)dnodeInitMain, (CleanupFp)dnodeCleanupMain); - taosStepAdd(steps, "dnode-storage", NULL, (InitFp)dnodeInitStorage, (CleanupFp)dnodeCleanupStorage); - //taosStepAdd(steps, "dnode-tfs", NULL, (InitFp)tfInit, (CleanupFp)tfCleanup); - taosStepAdd(steps, "dnode-rpc", NULL, (InitFp)rpcInit, (CleanupFp)rpcCleanup); - taosStepAdd(steps, "dnode-check", (void **)&dnode->check, (InitFp)dnodeInitCheck, (CleanupFp)dnodeCleanupCheck); - taosStepAdd(steps, "dnode-cfg", (void **)&dnode->cfg, (InitFp)dnodeInitCfg, (CleanupFp)dnodeCleanupCfg); - taosStepAdd(steps, "dnode-deps", (void **)&dnode->eps, (InitFp)dnodeInitEps, (CleanupFp)dnodeCleanupEps); - taosStepAdd(steps, "dnode-meps", (void **)&dnode->meps, (InitFp)dnodeInitMnodeEps, (CleanupFp)dnodeCleanupMnodeEps); - //taosStepAdd(steps, "dnode-wal", NULL, (InitFp)walInit, (CleanupFp)walCleanUp); - //taosStepAdd(steps, "dnode-sync", NULL, (InitFp)syncInit, (CleanupFp)syncCleanUp); - taosStepAdd(steps, "dnode-vnode", NULL, (InitFp)dnodeInitVnodeModule, (CleanupFp)vnodeCleanup); - taosStepAdd(steps, "dnode-mnode", NULL, (InitFp)dnodeInitMnodeModule, (CleanupFp)mnodeCleanup); - taosStepAdd(steps, "dnode-trans", (void **)&dnode->trans, (InitFp)dnodeInitTrans, (CleanupFp)dnodeCleanupTrans); - taosStepAdd(steps, "dnode-status", (void **)&dnode->status, (InitFp)dnodeInitStatus, (CleanupFp)dnodeCleanupStatus); - taosStepAdd(steps, "dnode-telem", (void **)&dnode->telem, (InitFp)dnodeInitTelem, (CleanupFp)dnodeCleanupTelem); - //taosStepAdd(steps, "dnode-script", NULL, (InitFp)scriptEnvPoolInit, (CleanupFp)scriptEnvPoolCleanup); + taosStepExec(tsSteps); - dnode->steps = steps; - taosStepExec(dnode->steps); - - if (dnode->main) { - dnode->main->runStatus = TD_RUN_STAT_RUNNING; - dnodeReportStartupFinished("TDengine", "initialized successfully"); - dInfo("TDengine is initialized successfully"); - } + dnodeSetRunStat(TD_RUN_STAT_RUNNING); + dnodeReportStartupFinished("TDengine", "initialized successfully"); + dInfo("TDengine is initialized successfully"); return 0; } void dnodeCleanup() { - SDnode *dnode = dnodeInst(); - if (dnode->main->runStatus != TD_RUN_STAT_STOPPED) { - dnode->main->runStatus = TD_RUN_STAT_STOPPED; - taosStepCleanup(dnode->steps); + if (dnodeGetRunStat() != TD_RUN_STAT_STOPPED) { + dnodeSetRunStat(TD_RUN_STAT_STOPPED); + taosStepCleanup(tsSteps); + tsSteps = NULL; } } diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c index 410cb41eed25938eeafe818a3c955b8be502e5ed..24de3b7924b4b01f8dcc2049fb60dfebae659896 100644 --- a/source/server/dnode/src/dnodeMain.c +++ b/source/server/dnode/src/dnodeMain.c @@ -17,15 +17,22 @@ #include "os.h" #include "tcache.h" #include "tconfig.h" +#include "tglobal.h" #if 0 #include "tfs.h" #endif +#include "tnote.h" +#include "tcompression.h" +#include "ttimer.h" #include "dnodeCfg.h" #include "dnodeMain.h" #include "mnode.h" -#include "tcompression.h" -#include "tnote.h" -#include "ttimer.h" + +static struct { + RunStat runStatus; + void * dnodeTimer; + SStartupStep startup; +} tsDmain; static void dnodeCheckDataDirOpenned(char *dir) { #if 0 @@ -47,27 +54,14 @@ static void dnodeCheckDataDirOpenned(char *dir) { #endif } -void dnodePrintDiskInfo() { - dInfo("=================================="); - dInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); - dInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); - dInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); - dInfo("=================================="); -} - -int32_t dnodeInitMain(SDnMain **out) { - SDnMain* main = calloc(1, sizeof(SDnMain)); - if (main == NULL) return -1; - - main->runStatus = TD_RUN_STAT_STOPPED; - main->dnodeTimer = taosTmrInit(100, 200, 60000, "DND-TMR"); - if (main->dnodeTimer == NULL) { +int32_t dnodeInitMain() { + tsDmain.runStatus = TD_RUN_STAT_STOPPED; + tsDmain.dnodeTimer = taosTmrInit(100, 200, 60000, "DND-TMR"); + if (tsDmain.dnodeTimer == NULL) { dError("failed to init dnode timer"); return -1; } - *out = main; - tscEmbedded = 1; taosIgnSIGPIPE(); taosBlockSIGPIPE(); @@ -76,7 +70,6 @@ int32_t dnodeInitMain(SDnMain **out) { taosReadGlobalLogCfg(); taosSetCoreDump(tsEnableCoreFile); - if (!taosMkDir(tsLogDir)) { printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); return -1; @@ -101,13 +94,10 @@ int32_t dnodeInitMain(SDnMain **out) { return taosCheckGlobalCfg(); } -void dnodeCleanupMain(SDnMain **out) { - SDnMain *main = *out; - *out = NULL; - - if (main->dnodeTimer != NULL) { - taosTmrCleanUp(main->dnodeTimer); - main->dnodeTimer = NULL; +void dnodeCleanupMain() { + if (tsDmain.dnodeTimer != NULL) { + taosTmrCleanUp(tsDmain.dnodeTimer); + tsDmain.dnodeTimer = NULL; } #if 0 @@ -115,8 +105,6 @@ void dnodeCleanupMain(SDnMain **out) { #endif taosCloseLog(); taosStopCacheRefreshWorker(); - - free(main); } int32_t dnodeInitStorage() { @@ -138,7 +126,7 @@ int32_t dnodeInitStorage() { } strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN); -#endif +#endif sprintf(tsMnodeDir, "%s/mnode", tsDataDir); sprintf(tsVnodeDir, "%s/vnode", tsDataDir); sprintf(tsDnodeDir, "%s/dnode", tsDataDir); @@ -164,7 +152,6 @@ int32_t dnodeInitStorage() { return -1; } - TDIR *tdir = tfsOpendir("vnode_bak/.staging"); bool stagingNotEmpty = tfsReaddir(tdir) != NULL; tfsClosedir(tdir); @@ -190,7 +177,7 @@ int32_t dnodeInitStorage() { } void dnodeCleanupStorage() { -#if 0 +#if 0 // storage destroy tfsDestroy(); @@ -202,18 +189,14 @@ void dnodeCleanupStorage() { } void dnodeReportStartup(char *name, char *desc) { - SDnode *dnode = dnodeInst(); - if (dnode->main != NULL) { - SStartupStep *startup = &dnode->main->startup; - tstrncpy(startup->name, name, strlen(startup->name)); - tstrncpy(startup->desc, desc, strlen(startup->desc)); - startup->finished = 0; - } + SStartupStep *startup = &tsDmain.startup; + tstrncpy(startup->name, name, strlen(startup->name)); + tstrncpy(startup->desc, desc, strlen(startup->desc)); + startup->finished = 0; } void dnodeReportStartupFinished(char *name, char *desc) { - SDnode *dnode = dnodeInst(); - SStartupStep *startup = &dnode->main->startup; + SStartupStep *startup = &tsDmain.startup; tstrncpy(startup->name, name, strlen(startup->name)); tstrncpy(startup->desc, desc, strlen(startup->desc)); startup->finished = 1; @@ -222,9 +205,8 @@ void dnodeReportStartupFinished(char *name, char *desc) { void dnodeProcessStartupReq(SRpcMsg *pMsg) { dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont); - SDnode *dnode = dnodeInst(); SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep)); - memcpy(pStep, &dnode->main->startup, sizeof(SStartupStep)); + memcpy(pStep, &tsDmain.startup, sizeof(SStartupStep)); dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished); @@ -234,12 +216,11 @@ void dnodeProcessStartupReq(SRpcMsg *pMsg) { } static int32_t dnodeStartMnode(SRpcMsg *pMsg) { - SDnode *dnode = dnodeInst(); SCreateMnodeMsg *pCfg = pMsg->pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); - if (pCfg->dnodeId != dnode->cfg->dnodeId) { + if (pCfg->dnodeId != dnodeGetDnodeId()) { dDebug("dnode:%d, in create meps msg is not equal with saved dnodeId:%d", pCfg->dnodeId, - dnodeGetDnodeId(dnode->cfg)); + dnodeGetDnodeId()); return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; } @@ -277,4 +258,10 @@ void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg) { rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); -} \ No newline at end of file +} + +RunStat dnodeGetRunStat() { return tsDmain.runStatus; } + +void dnodeSetRunStat(RunStat stat) { tsDmain.runStatus = stat; } + +void* dnodeGetTimer() { return tsDmain.dnodeTimer; } \ No newline at end of file diff --git a/source/server/dnode/src/dnodeMnodeEps.c b/source/server/dnode/src/dnodeMnodeEps.c index ab5c76b5806ecc3cb75d15a2cdb22755d5ee0e1a..5646fd5363984d63097b0b099fe9bbd9437bcbc2 100644 --- a/source/server/dnode/src/dnodeMnodeEps.c +++ b/source/server/dnode/src/dnodeMnodeEps.c @@ -22,43 +22,51 @@ #include "dnodeMnodeEps.h" #include "mnode.h" -static void dnodePrintMnodeEps(SDnMnEps *meps) { - SRpcEpSet *epset = &meps->mnodeEpSet; +static struct { + SRpcEpSet mnodeEpSet; + SMInfos mnodeInfos; + char file[PATH_MAX + 20]; + pthread_mutex_t mutex; +} tsDmeps; + + +static void dnodePrintMnodeEps() { + SRpcEpSet *epset = &tsDmeps.mnodeEpSet; dInfo("print mnode eps, num:%d inuse:%d", epset->numOfEps, epset->inUse); for (int32_t i = 0; i < epset->numOfEps; i++) { dInfo("ep index:%d, %s:%u", i, epset->fqdn[i], epset->port[i]); } } -static void dnodeResetMnodeEps(SDnMnEps *meps, SMInfos *mInfos) { +static void dnodeResetMnodeEps(SMInfos *mInfos) { if (mInfos == NULL || mInfos->mnodeNum == 0) { - meps->mnodeEpSet.numOfEps = 1; - taosGetFqdnPortFromEp(tsFirst, meps->mnodeEpSet.fqdn[0], &meps->mnodeEpSet.port[0]); + tsDmeps.mnodeEpSet.numOfEps = 1; + taosGetFqdnPortFromEp(tsFirst, tsDmeps.mnodeEpSet.fqdn[0], &tsDmeps.mnodeEpSet.port[0]); if (strcmp(tsSecond, tsFirst) != 0) { - meps->mnodeEpSet.numOfEps = 2; - taosGetFqdnPortFromEp(tsSecond, meps->mnodeEpSet.fqdn[1], &meps->mnodeEpSet.port[1]); + tsDmeps.mnodeEpSet.numOfEps = 2; + taosGetFqdnPortFromEp(tsSecond, tsDmeps.mnodeEpSet.fqdn[1], &tsDmeps.mnodeEpSet.port[1]); } - dnodePrintMnodeEps(meps); + dnodePrintMnodeEps(); return; } - int32_t size = sizeof(SMInfos); - memcpy(&meps->mnodeInfos, mInfos, size); + int32_t size = sizeof(SMInfos); + memcpy(&tsDmeps.mnodeInfos, mInfos, size); - meps->mnodeEpSet.inUse = meps->mnodeInfos.inUse; - meps->mnodeEpSet.numOfEps = meps->mnodeInfos.mnodeNum; - for (int32_t i = 0; i < meps->mnodeInfos.mnodeNum; i++) { - taosGetFqdnPortFromEp(meps->mnodeInfos.mnodeInfos[i].mnodeEp, meps->mnodeEpSet.fqdn[i], &meps->mnodeEpSet.port[i]); + tsDmeps.mnodeEpSet.inUse = tsDmeps.mnodeInfos.inUse; + tsDmeps.mnodeEpSet.numOfEps = tsDmeps.mnodeInfos.mnodeNum; + for (int32_t i = 0; i < tsDmeps.mnodeInfos.mnodeNum; i++) { + taosGetFqdnPortFromEp(tsDmeps.mnodeInfos.mnodeInfos[i].mnodeEp, tsDmeps.mnodeEpSet.fqdn[i], &tsDmeps.mnodeEpSet.port[i]); } - dnodePrintMnodeEps(meps); + dnodePrintMnodeEps(); } -static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) { - FILE *fp = fopen(meps->file, "w"); +static int32_t dnodeWriteMnodeEps() { + FILE *fp = fopen(tsDmeps.file, "w"); if (!fp) { - dError("failed to write %s since %s", meps->file, strerror(errno)); + dError("failed to write %s since %s", tsDmeps.file, strerror(errno)); return -1; } @@ -67,13 +75,13 @@ static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) { char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", meps->mnodeInfos.inUse); - len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", meps->mnodeInfos.mnodeNum); + len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsDmeps.mnodeInfos.inUse); + len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsDmeps.mnodeInfos.mnodeNum); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < meps->mnodeInfos.mnodeNum; i++) { - len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", meps->mnodeInfos.mnodeInfos[i].mnodeId); - len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", meps->mnodeInfos.mnodeInfos[i].mnodeEp); - if (i < meps->mnodeInfos.mnodeNum - 1) { + for (int32_t i = 0; i < tsDmeps.mnodeInfos.mnodeNum; i++) { + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsDmeps.mnodeInfos.mnodeInfos[i].mnodeId); + len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsDmeps.mnodeInfos.mnodeInfos[i].mnodeEp); + if (i < tsDmeps.mnodeInfos.mnodeNum - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { len += snprintf(content + len, maxLen - len, " }]\n"); @@ -87,11 +95,11 @@ static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) { free(content); terrno = 0; - dInfo("successed to write %s", meps->file); + dInfo("successed to write %s", tsDmeps.file); return 0; } -static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) { +static int32_t dnodeReadMnodeEps() { int32_t len = 0; int32_t maxLen = 2000; char * content = calloc(1, maxLen + 1); @@ -100,22 +108,22 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) { SMInfos mInfos = {0}; bool nodeChanged = false; - fp = fopen(meps->file, "r"); + fp = fopen(tsDmeps.file, "r"); if (!fp) { - dDebug("file %s not exist", meps->file); + dDebug("file %s not exist", tsDmeps.file); goto PARSE_MINFOS_OVER; } len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { - dError("failed to read %s since content is null", meps->file); + dError("failed to read %s since content is null", tsDmeps.file); goto PARSE_MINFOS_OVER; } content[len] = 0; root = cJSON_Parse(content); if (root == NULL) { - dError("failed to read %s since invalid json format", meps->file); + dError("failed to read %s since invalid json format", tsDmeps.file); goto PARSE_MINFOS_OVER; } @@ -124,7 +132,7 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) { dError("failed to read mnodeEpSet.json since inUse not found"); goto PARSE_MINFOS_OVER; } - meps->mnodeInfos.inUse = (int8_t)inUse->valueint; + tsDmeps.mnodeInfos.inUse = (int8_t)inUse->valueint; cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { @@ -165,11 +173,11 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) { mInfo->mnodeId = (int32_t)nodeId->valueint; tstrncpy(mInfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); - bool changed = dnodeIsDnodeEpChanged(deps, mInfo->mnodeId, mInfo->mnodeEp); + bool changed = dnodeIsDnodeEpChanged(mInfo->mnodeId, mInfo->mnodeEp); if (changed) nodeChanged = changed; } - dInfo("successed to read file %s", meps->file); + dInfo("successed to read file %s", tsDmeps.file); PARSE_MINFOS_OVER: if (content != NULL) free(content); @@ -182,25 +190,24 @@ PARSE_MINFOS_OVER: dnodeGetDnodeEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); } - dnodeResetMnodeEps(meps, &mInfos); + dnodeResetMnodeEps(&mInfos); if (nodeChanged) { - dnodeWriteMnodeEps(meps); + dnodeWriteMnodeEps(); } return 0; } void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { - SDnMnEps *meps = dnodeInst()->meps; SRpcConnInfo connInfo = {0}; rpcGetConnInfo(rpcMsg->handle, &connInfo); SRpcEpSet epSet = {0}; if (forShell) { - dnodeGetEpSetForShell(meps, &epSet); + dnodeGetEpSetForShell(&epSet); } else { - dnodeGetEpSetForPeer(meps, &epSet); + dnodeGetEpSetForPeer(&epSet); } dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], @@ -222,16 +229,12 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { rpcSendRedirectRsp(rpcMsg->handle, &epSet); } -int32_t dnodeInitMnodeEps(SDnMnEps **out) { - SDnMnEps *meps = calloc(1, sizeof(SDnMnEps)); - if (meps == NULL) return -1; - - snprintf(meps->file, sizeof(meps->file), "%s/mnodeEpSet.json", tsDnodeDir); - pthread_mutex_init(&meps->mutex, NULL); - *out = meps; +int32_t dnodeInitMnodeEps() { + snprintf(tsDmeps.file, sizeof(tsDmeps.file), "%s/mnodeEpSet.json", tsDnodeDir); + pthread_mutex_init(&tsDmeps.mutex, NULL); - dnodeResetMnodeEps(meps, NULL); - int32_t ret = dnodeReadMnodeEps(meps, dnodeInst()->eps); + dnodeResetMnodeEps(NULL); + int32_t ret = dnodeReadMnodeEps(); if (ret == 0) { dInfo("dnode mInfos is initialized"); } @@ -239,17 +242,11 @@ int32_t dnodeInitMnodeEps(SDnMnEps **out) { return ret; } -void dnodeCleanupMnodeEps(SDnMnEps **out) { - SDnMnEps *meps = *out; - *out = NULL; - - if (meps != NULL) { - pthread_mutex_destroy(&meps->mutex); - free(meps); - } +void dnodeCleanupMnodeEps() { + pthread_mutex_destroy(&tsDmeps.mutex); } -void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *mInfos) { +void dnodeUpdateMnodeFromStatus(SMInfos *mInfos) { if (mInfos->mnodeNum <= 0 || mInfos->mnodeNum > TSDB_MAX_REPLICA) { dError("invalid mInfos since num:%d invalid", mInfos->mnodeNum); return; @@ -264,53 +261,51 @@ void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *mInfos) { } } - pthread_mutex_lock(&meps->mutex); - if (mInfos->mnodeNum != meps->mnodeInfos.mnodeNum) { - dnodeResetMnodeEps(meps, mInfos); - dnodeWriteMnodeEps(meps); + pthread_mutex_lock(&tsDmeps.mutex); + if (mInfos->mnodeNum != tsDmeps.mnodeInfos.mnodeNum) { + dnodeResetMnodeEps(mInfos); + dnodeWriteMnodeEps(); } else { int32_t size = sizeof(SMInfos); - if (memcmp(mInfos, &meps->mnodeInfos, size) != 0) { - dnodeResetMnodeEps(meps, mInfos); - dnodeWriteMnodeEps(meps); + if (memcmp(mInfos, &tsDmeps.mnodeInfos, size) != 0) { + dnodeResetMnodeEps(mInfos); + dnodeWriteMnodeEps(); } } - pthread_mutex_unlock(&meps->mutex); + pthread_mutex_unlock(&tsDmeps.mutex); } -void dnodeUpdateMnodeFromPeer(SDnMnEps *meps, SRpcEpSet *ep) { +void dnodeUpdateMnodeFromPeer(SRpcEpSet *ep) { if (ep->numOfEps <= 0) { dError("mInfos is changed, but content is invalid, discard it"); return; } - pthread_mutex_lock(&meps->mutex); + pthread_mutex_lock(&tsDmeps.mutex); dInfo("mInfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); for (int32_t i = 0; i < ep->numOfEps; ++i) { ep->port[i] -= TSDB_PORT_DNODEDNODE; dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]); } - meps->mnodeEpSet = *ep; + tsDmeps.mnodeEpSet = *ep; - pthread_mutex_unlock(&meps->mutex); + pthread_mutex_unlock(&tsDmeps.mutex); } -void dnodeGetEpSetForPeer(SDnMnEps *meps, SRpcEpSet *epSet) { - pthread_mutex_lock(&meps->mutex); +void dnodeGetEpSetForPeer(SRpcEpSet *epSet) { + pthread_mutex_lock(&tsDmeps.mutex); - *epSet = meps->mnodeEpSet; + *epSet = tsDmeps.mnodeEpSet; for (int32_t i = 0; i < epSet->numOfEps; ++i) { epSet->port[i] += TSDB_PORT_DNODEDNODE; } - pthread_mutex_unlock(&meps->mutex); + pthread_mutex_unlock(&tsDmeps.mutex); } -void dnodeGetEpSetForShell(SDnMnEps *meps, SRpcEpSet *epSet) { - pthread_mutex_lock(&meps->mutex); - - *epSet = meps->mnodeEpSet; - - pthread_mutex_unlock(&meps->mutex); +void dnodeGetEpSetForShell(SRpcEpSet *epSet) { + pthread_mutex_lock(&tsDmeps.mutex); + *epSet = tsDmeps.mnodeEpSet; + pthread_mutex_unlock(&tsDmeps.mutex); } diff --git a/source/server/dnode/src/dnodeStatus.c b/source/server/dnode/src/dnodeStatus.c index 6abc886147ae4d98455e5dbe1305182703f9cf56..38b685c1fb3d11915fed28d5c1621b83ed4376c9 100644 --- a/source/server/dnode/src/dnodeStatus.c +++ b/source/server/dnode/src/dnodeStatus.c @@ -25,15 +25,15 @@ #include "dnodeMain.h" #include "vnode.h" -static void dnodeSendStatusMsg(void *handle, void *tmrId) { - SDnStatus *status = handle; - if (status->dnodeTimer == NULL) { - dError("dnode timer is already released"); - return; - } +static struct { + void * dnodeTimer; + void * statusTimer; + uint32_t rebootTime; +} tsStatus; - if (status->statusTimer == NULL) { - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); +static void dnodeSendStatusMsg(void *handle, void *tmrId) { + if (tsStatus.statusTimer == NULL) { + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer); dError("failed to start status timer"); return; } @@ -41,16 +41,15 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); SStatusMsg *pStatus = rpcMallocCont(contLen); if (pStatus == NULL) { - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer); dError("failed to malloc status message"); return; } - SDnode *dnode = dnodeInst(); - dnodeGetCfg(dnode->cfg, &pStatus->dnodeId, pStatus->clusterId); - pStatus->dnodeId = htonl(dnodeGetDnodeId(dnode->cfg)); + dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId); + pStatus->dnodeId = htonl(dnodeGetDnodeId()); pStatus->version = htonl(tsVersion); - pStatus->lastReboot = htonl(status->rebootTime); + pStatus->lastReboot = htonl(tsStatus.rebootTime); pStatus->numOfCores = htons((uint16_t)tsNumOfCores); pStatus->diskAvailable = tsAvailDataDirGB; pStatus->alternativeRole = tsAlternativeRole; @@ -80,69 +79,58 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); pStatus->openVnodes = htons(pStatus->openVnodes); - SRpcMsg rpcMsg = {.ahandle = status, .pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_DM_STATUS}; + SRpcMsg rpcMsg = {.ahandle = NULL, .pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_DM_STATUS}; dnodeSendMsgToMnode(&rpcMsg); } void dnodeProcessStatusRsp(SRpcMsg *pMsg) { - SDnode *dnode = dnodeInst(); - SDnStatus *status = pMsg->ahandle; - if (pMsg->code != TSDB_CODE_SUCCESS) { dError("status rsp is received, error:%s", tstrerror(pMsg->code)); if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST) { char clusterId[TSDB_CLUSTER_ID_LEN]; - dnodeGetClusterId(dnode->cfg, clusterId); + dnodeGetClusterId(clusterId); if (clusterId[0] != '\0') { - dnodeSetDropped(dnode->cfg); + dnodeSetDropped(); dError("exit zombie dropped dnode"); exit(EXIT_FAILURE); } } - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer); return; } SStatusRsp *pStatusRsp = pMsg->pCont; SMInfos * minfos = &pStatusRsp->mnodes; - dnodeUpdateMnodeFromStatus(dnode->meps, minfos); + dnodeUpdateMnodeFromStatus(minfos); SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); pCfg->moduleStatus = htonl(pCfg->moduleStatus); pCfg->dnodeId = htonl(pCfg->dnodeId); - dnodeUpdateCfg(dnode->cfg, pCfg); + dnodeUpdateCfg(pCfg); vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes); SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess)); - dnodeUpdateEps(dnode->eps, pEps); + dnodeUpdateEps(pEps); - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer); } -int32_t dnodeInitStatus(SDnStatus **out) { - SDnStatus *status = calloc(1, sizeof(SDnStatus)); - if (status == NULL) return -1; - status->statusTimer = NULL; - status->dnodeTimer = dnodeInst()->main->dnodeTimer; - status->rebootTime = taosGetTimestampSec(); - taosTmrReset(dnodeSendStatusMsg, 500, status, status->dnodeTimer, &status->statusTimer); - *out = status; +int32_t dnodeInitStatus() { + tsStatus.statusTimer = NULL; + tsStatus.dnodeTimer = dnodeGetTimer(); + tsStatus.rebootTime = taosGetTimestampSec(); + taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer); dInfo("dnode status timer is initialized"); return TSDB_CODE_SUCCESS; } -void dnodeCleanupStatus(SDnStatus **out) { - SDnStatus *status = *out; - *out = NULL; - - if (status->statusTimer != NULL) { - taosTmrStopA(&status->statusTimer); - status->statusTimer = NULL; +void dnodeCleanupStatus() { + if (tsStatus.statusTimer != NULL) { + taosTmrStopA(&tsStatus.statusTimer); + tsStatus.statusTimer = NULL; } - - free(status); } diff --git a/source/server/dnode/src/dnodeTelem.c b/source/server/dnode/src/dnodeTelem.c index b221746c833b4f79401275745e1ae2c2b867a361..7c87ea5f50fedfdf8d8102746dd2bfb28e4a4261 100644 --- a/source/server/dnode/src/dnodeTelem.c +++ b/source/server/dnode/src/dnodeTelem.c @@ -25,6 +25,19 @@ #define TELEMETRY_PORT 80 #define REPORT_INTERVAL 86400 +/* + * sem_timedwait is NOT implemented on MacOSX + * thus we use pthread_mutex_t/pthread_cond_t to simulate + */ +static struct { + bool enable; + pthread_mutex_t lock; + pthread_cond_t cond; + volatile int32_t exit; + pthread_t thread; + char email[TSDB_FQDN_LEN]; +} tsTelem; + static void dnodeBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } static void dnodeCloseObject(SBufferWriter* bw) { @@ -154,14 +167,14 @@ static void dnodeAddMemoryInfo(SBufferWriter* bw) { fclose(fp); } -static void dnodeAddVersionInfo(SDnTelem* telem, SBufferWriter* bw) { +static void dnodeAddVersionInfo(SBufferWriter* bw) { dnodeAddStringField(bw, "version", version); dnodeAddStringField(bw, "buildInfo", buildinfo); dnodeAddStringField(bw, "gitInfo", gitinfo); - dnodeAddStringField(bw, "email", telem->email); + dnodeAddStringField(bw, "email", tsTelem.email); } -static void dnodeAddRuntimeInfo(SDnTelem* telem, SBufferWriter* bw) { +static void dnodeAddRuntimeInfo(SBufferWriter* bw) { SMnodeStat stat = {0}; if (mnodeGetStatistics(&stat) != 0) { return; @@ -179,7 +192,7 @@ static void dnodeAddRuntimeInfo(SDnTelem* telem, SBufferWriter* bw) { dnodeAddIntField(bw, "compStorage", stat.compStorage); } -static void dnodeSendTelemetryReport(SDnTelem* telem) { +static void dnodeSendTelemetryReport() { char buf[128] = {0}; uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { @@ -192,16 +205,18 @@ static void dnodeSendTelemetryReport(SDnTelem* telem) { return; } - SDnode *dnode = dnodeInst(); + char clusterId[TSDB_CLUSTER_ID_LEN] = {0}; + dnodeGetClusterId(clusterId); + SBufferWriter bw = tbufInitWriter(NULL, false); dnodeBeginObject(&bw); - dnodeAddStringField(&bw, "instanceId", dnode->cfg->clusterId); + dnodeAddStringField(&bw, "instanceId", clusterId); dnodeAddIntField(&bw, "reportVersion", 1); dnodeAddOsInfo(&bw); dnodeAddCpuInfo(&bw); dnodeAddMemoryInfo(&bw); - dnodeAddVersionInfo(telem, &bw); - dnodeAddRuntimeInfo(telem, &bw); + dnodeAddVersionInfo(&bw); + dnodeAddRuntimeInfo(&bw); dnodeCloseObject(&bw); const char* header = @@ -227,25 +242,23 @@ static void dnodeSendTelemetryReport(SDnTelem* telem) { } static void* dnodeTelemThreadFp(void* param) { - SDnTelem* telem = param; - struct timespec end = {0}; clock_gettime(CLOCK_REALTIME, &end); end.tv_sec += 300; // wait 5 minutes before send first report setThreadName("dnode-telem"); - while (!telem->exit) { + while (!tsTelem.exit) { int32_t r = 0; struct timespec ts = end; - pthread_mutex_lock(&telem->lock); - r = pthread_cond_timedwait(&telem->cond, &telem->lock, &ts); - pthread_mutex_unlock(&telem->lock); + pthread_mutex_lock(&tsTelem.lock); + r = pthread_cond_timedwait(&tsTelem.cond, &tsTelem.lock, &ts); + pthread_mutex_unlock(&tsTelem.lock); if (r == 0) break; if (r != ETIMEDOUT) continue; if (mnodeIsServing()) { - dnodeSendTelemetryReport(telem); + dnodeSendTelemetryReport(); } end.tv_sec += REPORT_INTERVAL; } @@ -253,40 +266,35 @@ static void* dnodeTelemThreadFp(void* param) { return NULL; } -static void dnodeGetEmail(SDnTelem* telem, char* filepath) { +static void dnodeGetEmail(char* filepath) { int32_t fd = taosOpenFileRead(filepath); if (fd < 0) { return; } - if (taosReadFile(fd, (void*)telem->email, TSDB_FQDN_LEN) < 0) { + if (taosReadFile(fd, (void*)tsTelem.email, TSDB_FQDN_LEN) < 0) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } taosCloseFile(fd); } -int32_t dnodeInitTelem(SDnTelem** out) { - SDnTelem* telem = calloc(1, sizeof(SDnTelem)); - if (telem == NULL) return -1; - - telem->enable = tsEnableTelemetryReporting; - *out = telem; +int32_t dnodeInitTelem() { + tsTelem.enable = tsEnableTelemetryReporting; + if (!tsTelem.enable) return 0; - if (!telem->enable) return 0; + tsTelem.exit = 0; + pthread_mutex_init(&tsTelem.lock, NULL); + pthread_cond_init(&tsTelem.cond, NULL); + tsTelem.email[0] = 0; - telem->exit = 0; - pthread_mutex_init(&telem->lock, NULL); - pthread_cond_init(&telem->cond, NULL); - telem->email[0] = 0; - - dnodeGetEmail(telem, "/usr/local/taos/email"); + dnodeGetEmail("/usr/local/taos/email"); pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int32_t code = pthread_create(&telem->thread, &attr, dnodeTelemThreadFp, telem); + int32_t code = pthread_create(&tsTelem.thread, &attr, dnodeTelemThreadFp, NULL); pthread_attr_destroy(&attr); if (code != 0) { dTrace("failed to create telemetry thread since :%s", strerror(code)); @@ -296,26 +304,18 @@ int32_t dnodeInitTelem(SDnTelem** out) { return 0; } -void dnodeCleanupTelem(SDnTelem** out) { - SDnTelem* telem = *out; - *out = NULL; +void dnodeCleanupTelem() { + if (!tsTelem.enable) return; - if (!telem->enable) { - free(telem); - return; - } + if (taosCheckPthreadValid(tsTelem.thread)) { + pthread_mutex_lock(&tsTelem.lock); + tsTelem.exit = 1; + pthread_cond_signal(&tsTelem.cond); + pthread_mutex_unlock(&tsTelem.lock); - if (taosCheckPthreadValid(telem->thread)) { - pthread_mutex_lock(&telem->lock); - telem->exit = 1; - pthread_cond_signal(&telem->cond); - pthread_mutex_unlock(&telem->lock); - - pthread_join(telem->thread, NULL); + pthread_join(tsTelem.thread, NULL); } - pthread_mutex_destroy(&telem->lock); - pthread_cond_destroy(&telem->cond); - - free(telem); + pthread_mutex_destroy(&tsTelem.lock); + pthread_cond_destroy(&tsTelem.cond); } diff --git a/source/server/dnode/src/dnodeTrans.c b/source/server/dnode/src/dnodeTrans.c index 7a870f22c85ed59c10401e98d57e9324d81f0193..a4409674f1d039e4478f78a4b7bc0b0d17ea2b3d 100644 --- a/source/server/dnode/src/dnodeTrans.c +++ b/source/server/dnode/src/dnodeTrans.c @@ -14,7 +14,7 @@ */ /* this file is mainly responsible for the communication between DNODEs. Each - * dnode works as both server and client. SDnode may send status, grant, config + * dnode works as both server and client. Dnode may send status, grant, config * messages to mnode, mnode may send create/alter/drop table/vnode messages * to dnode. All theses messages are handled from here */ @@ -29,8 +29,19 @@ #include "vnode.h" #include "mnode.h" +typedef void (*RpcMsgFp)( SRpcMsg *pMsg); + +static struct { + void * serverRpc; + void * clientRpc; + void * shellRpc; + int32_t queryReqNum; + int32_t submitReqNum; + RpcMsgFp peerMsgFp[TSDB_MSG_TYPE_MAX]; + RpcMsgFp shellMsgFp[TSDB_MSG_TYPE_MAX]; +} tsTrans; + static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SDnode * dnode = dnodeInst(); SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; if (pMsg->pCont == NULL) return; @@ -39,7 +50,7 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { return; } - if (dnode->main->runStatus != TD_RUN_STAT_RUNNING) { + if (dnodeGetRunStat() != TD_RUN_STAT_RUNNING) { rspMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); @@ -53,7 +64,7 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { return; } - RpcMsgFp fp = dnode->trans->peerMsgFp[pMsg->msgType]; + RpcMsgFp fp = tsTrans.peerMsgFp[pMsg->msgType]; if (fp != NULL) { (*fp)(pMsg); } else { @@ -64,27 +75,27 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } } -int32_t dnodeInitServer(SDnTrans *trans) { - trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessMsg; +int32_t dnodeInitServer() { + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = vnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = vnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeReq; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeReq; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeReq; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeReq; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_AUTH] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_GRANT] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_STATUS] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_AUTH] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_GRANT] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_STATUS] = mnodeProcessMsg; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); @@ -96,8 +107,8 @@ int32_t dnodeInitServer(SDnTrans *trans) { rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - trans->serverRpc = rpcOpen(&rpcInit); - if (trans->serverRpc == NULL) { + tsTrans.serverRpc = rpcOpen(&rpcInit); + if (tsTrans.serverRpc == NULL) { dError("failed to init peer rpc server"); return -1; } @@ -106,17 +117,16 @@ int32_t dnodeInitServer(SDnTrans *trans) { return 0; } -void dnodeCleanupServer(SDnTrans *trans) { - if (trans->serverRpc) { - rpcClose(trans->serverRpc); - trans->serverRpc = NULL; +void dnodeCleanupServer() { + if (tsTrans.serverRpc) { + rpcClose(tsTrans.serverRpc); + tsTrans.serverRpc = NULL; dInfo("dnode peer server is closed"); } } static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SDnode *dnode = dnodeInst(); - if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) { + if (dnodeGetRunStat() == TD_RUN_STAT_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; dTrace("msg:%p is ignored since dnode is stopping", pMsg); rpcFreeCont(pMsg->pCont); @@ -124,10 +134,10 @@ static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) { - dnodeUpdateMnodeFromPeer(dnode->meps, pEpSet); + dnodeUpdateMnodeFromPeer(pEpSet); } - RpcMsgFp fp = dnode->trans->peerMsgFp[pMsg->msgType]; + RpcMsgFp fp = tsTrans.peerMsgFp[pMsg->msgType]; if (fp != NULL) { (*fp)(pMsg); } else { @@ -141,27 +151,27 @@ static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { rpcFreeCont(pMsg->pCont); } -int32_t dnodeInitClient(SDnTrans *trans) { - trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP] = mnodeProcessMsg; +int32_t dnodeInitClient() { + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_AUTH_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_GRANT_RSP] = mnodeProcessMsg; - trans->peerMsgFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_AUTH_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_GRANT_RSP] = mnodeProcessMsg; + tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp; char secret[TSDB_KEY_LEN] = "secret"; SRpcInit rpcInit; @@ -176,8 +186,8 @@ int32_t dnodeInitClient(SDnTrans *trans) { rpcInit.ckey = "key"; rpcInit.secret = secret; - trans->clientRpc = rpcOpen(&rpcInit); - if (trans->clientRpc == NULL) { + tsTrans.clientRpc = rpcOpen(&rpcInit); + if (tsTrans.clientRpc == NULL) { dError("failed to init peer rpc client"); return -1; } @@ -186,26 +196,25 @@ int32_t dnodeInitClient(SDnTrans *trans) { return 0; } -void dnodeCleanupClient(SDnTrans *trans) { - if (trans->clientRpc) { - rpcClose(trans->clientRpc); - trans->clientRpc = NULL; +void dnodeCleanupClient() { + if (tsTrans.clientRpc) { + rpcClose(tsTrans.clientRpc); + tsTrans.clientRpc = NULL; dInfo("dnode peer rpc client is closed"); } } static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SDnode * dnode = dnodeInst(); SRpcMsg rpcMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; if (pMsg->pCont == NULL) return; - if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) { + if (dnodeGetRunStat() == TD_RUN_STAT_STOPPED) { dError("RPC %p, shell msg:%s is ignored since dnode exiting", pMsg->handle, taosMsg[pMsg->msgType]); rpcMsg.code = TSDB_CODE_DND_EXITING; rpcSendResponse(&rpcMsg); rpcFreeCont(pMsg->pCont); return; - } else if (dnode->main->runStatus != TD_RUN_STAT_RUNNING) { + } else if (dnodeGetRunStat() != TD_RUN_STAT_RUNNING) { dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); rpcMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rpcMsg); @@ -213,14 +222,13 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { return; } - SDnTrans *trans = dnode->trans; if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { - atomic_fetch_add_32(&trans->queryReqNum, 1); + atomic_fetch_add_32(&tsTrans.queryReqNum, 1); } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { - atomic_fetch_add_32(&trans->submitReqNum, 1); + atomic_fetch_add_32(&tsTrans.submitReqNum, 1); } else {} - RpcMsgFp fp = trans->shellMsgFp[pMsg->msgType]; + RpcMsgFp fp = tsTrans.shellMsgFp[pMsg->msgType]; if (fp != NULL) { (*fp)(pMsg); } else { @@ -247,27 +255,23 @@ static int32_t dnodeAuthNetTest(char *user, char *spi, char *encrypt, char *secr } void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { - SDnode *dnode = dnodeInst(); - rpcSendRequest(dnode->trans->clientRpc, epSet, rpcMsg, NULL); + rpcSendRequest(tsTrans.clientRpc, epSet, rpcMsg, NULL); } void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { - SDnode * dnode = dnodeInst(); SRpcEpSet epSet = {0}; - dnodeGetEpSetForPeer(dnode->meps, &epSet); + dnodeGetEpSetForPeer(&epSet); dnodeSendMsgToDnode(&epSet, rpcMsg); } void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { - SDnode * dnode = dnodeInst(); SRpcEpSet epSet = {0}; - dnodeGetEpSetForPeer(dnode->meps, &epSet); - rpcSendRecv(dnode->trans->clientRpc, &epSet, rpcMsg, rpcRsp); + dnodeGetEpSetForPeer(&epSet); + rpcSendRecv(tsTrans.clientRpc, &epSet, rpcMsg, rpcRsp); } void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) { - SDnode *dnode = dnodeInst(); - rpcSendRecv(dnode->trans->clientRpc, epSet, rpcMsg, rpcRsp); + rpcSendRecv(tsTrans.clientRpc, epSet, rpcMsg, rpcRsp); } static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { @@ -303,52 +307,52 @@ static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, c return rpcRsp.code; } -int32_t dnodeInitShell(SDnTrans *trans) { - trans->shellMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessMsg; +int32_t dnodeInitShell() { + tsTrans.shellMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessMsg; // the following message shall be treated as mnode write - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_COMPACT_VNODE] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_COMPACT_VNODE] = mnodeProcessMsg; // the following message shall be treated as mnode query - trans->shellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = mnodeProcessMsg; - trans->shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = mnodeProcessMsg; - - trans->shellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeProcessStartupReq; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = mnodeProcessMsg; + tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = mnodeProcessMsg; + + tsTrans.shellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeProcessStartupReq; int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { @@ -366,8 +370,8 @@ int32_t dnodeInitShell(SDnTrans *trans) { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = dnodeRetrieveUserAuthInfo; - trans->shellRpc = rpcOpen(&rpcInit); - if (trans->shellRpc == NULL) { + tsTrans.shellRpc = rpcOpen(&rpcInit); + if (tsTrans.shellRpc == NULL) { dError("failed to init shell rpc server"); return -1; } @@ -376,41 +380,31 @@ int32_t dnodeInitShell(SDnTrans *trans) { return 0; } -void dnodeCleanupShell(SDnTrans *trans) { - if (trans->shellRpc) { - rpcClose(trans->shellRpc); - trans->shellRpc = NULL; +void dnodeCleanupShell() { + if (tsTrans.shellRpc) { + rpcClose(tsTrans.shellRpc); + tsTrans.shellRpc = NULL; } } -int32_t dnodeInitTrans(SDnTrans **out) { - SDnTrans *trans = calloc(1, sizeof(SDnTrans)); - if (trans == NULL) return -1; - - *out = trans; - - if (dnodeInitClient(trans) != 0) { +int32_t dnodeInitTrans() { + if (dnodeInitClient() != 0) { return -1; } - if (dnodeInitServer(trans) != 0) { + if (dnodeInitServer() != 0) { return -1; } - if (dnodeInitShell(trans) != 0) { + if (dnodeInitShell() != 0) { return -1; } return 0; } -void dnodeCleanupTrans(SDnTrans **out) { - SDnTrans* trans = *out; - *out = NULL; - - dnodeCleanupShell(trans); - dnodeCleanupServer(trans); - dnodeCleanupClient(trans); - - free(trans); +void dnodeCleanupTrans() { + dnodeCleanupShell(); + dnodeCleanupServer(); + dnodeCleanupClient(); } diff --git a/source/server/vnode/CMakeLists.txt b/source/server/vnode/CMakeLists.txt index 5e11e4556798d5ed56776b69ebf530617fa27f6e..249f56657c1a6798085ca93087c3e91065b6b035 100644 --- a/source/server/vnode/CMakeLists.txt +++ b/source/server/vnode/CMakeLists.txt @@ -15,9 +15,11 @@ target_link_libraries( PUBLIC meta PUBLIC tq PUBLIC tsdb + PUBLIC wal + PUBLIC cjson ) # test if(${BUILD_TEST}) add_subdirectory(test) -endif(${BUILD_TEST}) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/src/vnode/inc/vnodeCfg.h b/source/server/vnode/inc/vnodeCfg.h similarity index 86% rename from src/vnode/inc/vnodeCfg.h rename to source/server/vnode/inc/vnodeCfg.h index ba148c07c1e4f9451232c706fb79c0af0cad9746..342d801f44411076deacc300d2305d1deb80b20b 100644 --- a/src/vnode/inc/vnodeCfg.h +++ b/source/server/vnode/inc/vnodeCfg.h @@ -13,19 +13,19 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_CFG_H -#define TDENGINE_VNODE_CFG_H +#ifndef _TD_VNODE_CFG_H_ +#define _TD_VNODE_CFG_H_ #ifdef __cplusplus extern "C" { #endif #include "vnodeInt.h" -int32_t vnodeReadCfg(SVnodeObj *pVnode); +int32_t vnodeReadCfg(SVnode *pVnode); int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg); #ifdef __cplusplus } #endif -#endif +#endif /*_TD_VNODE_CFG_H_*/ diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 545d376f498c7e23efa615b3762a81ab2c36bea0..5f07c5819c7f357a749ca358ce4652720adf1ece 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -16,7 +16,6 @@ #ifndef _TD_VNODE_INT_H_ #define _TD_VNODE_INT_H_ - #include "os.h" #include "amalloc.h" #include "meta.h" @@ -25,20 +24,83 @@ #include "trpc.h" #include "tsdb.h" #include "vnode.h" +#include "tlog.h" +#include "tqueue.h" +#include "wal.h" +#include "tworker.h" #ifdef __cplusplus extern "C" { #endif -typedef struct SVnode { +extern int32_t vDebugFlag; + +#define vFatal(...) { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }} +#define vError(...) { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }} +#define vWarn(...) { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }} +#define vInfo(...) { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("VND ", 255, __VA_ARGS__); }} +#define vDebug(...) { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }} +#define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }} + +typedef struct { SMeta * pMeta; STsdb * pTsdb; STQ * pTQ; SMemAllocator *allocator; + + int32_t vgId; // global vnode group ID + int32_t refCount; // reference count + int64_t queuedWMsgSize; + int32_t queuedWMsg; + int32_t queuedRMsg; + int32_t numOfExistQHandle; // current initialized and existed query handle in current dnode + int32_t flowctrlLevel; + int8_t preClose; // drop and close switch + int8_t reserved[3]; + int64_t sequence; // for topic + int8_t status; + int8_t role; + int8_t accessState; + int8_t isFull; + int8_t isCommiting; + int8_t dbReplica; + int8_t dropped; + int8_t dbType; + uint64_t version; // current version + uint64_t cversion; // version while commit start + uint64_t fversion; // version on saved data file + void * wqueue; // write queue + void * qqueue; // read query queue + void * fqueue; // read fetch/cancel queue + void * wal; + void * tsdb; + int64_t sync; + void * events; + void * cq; // continuous query + int32_t dbCfgVersion; + int32_t vgCfgVersion; + STsdbCfg tsdbCfg; +#if 0 + SSyncCfg syncCfg; +#endif + SWalCfg walCfg; + void * qMgmt; + char * rootDir; + tsem_t sem; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + pthread_mutex_t statusMutex; } SVnode; +typedef struct { + int32_t len; + void * rsp; + void * qhandle; // used by query and retrieve msg +} SVnRsp; + +void vnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); + #ifdef __cplusplus } #endif -#endif /*_TD_VNODE_INT_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_INT_H_*/ diff --git a/src/vnode/inc/vnodeMain.h b/source/server/vnode/inc/vnodeMain.h similarity index 63% rename from src/vnode/inc/vnodeMain.h rename to source/server/vnode/inc/vnodeMain.h index 91a5d632cd64d7979b77dcf86472ddf3be2aa1b4..093d07b013415aff424c2efb33507b3f99264697 100644 --- a/src/vnode/inc/vnodeMain.h +++ b/source/server/vnode/inc/vnodeMain.h @@ -13,25 +13,35 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_MAIN_H -#define TDENGINE_VNODE_MAIN_H +#ifndef _TD_VNODE_MAIN_H_ +#define _TD_VNODE_MAIN_H_ #ifdef __cplusplus extern "C" { #endif #include "vnodeInt.h" +int32_t vnodeInitMain(); +void vnodeCleanupMain(); + int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId); -int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); +int32_t vnodeAlter(SVnode *pVnode, SCreateVnodeMsg *pVnodeCfg); int32_t vnodeSync(int32_t vgId); int32_t vnodeClose(int32_t vgId); -void vnodeCleanUp(SVnodeObj *pVnode); -void vnodeDestroy(SVnodeObj *pVnode); +void vnodeCleanUp(SVnode *pVnode); +void vnodeDestroy(SVnode *pVnode); +int32_t vnodeCompact(int32_t vgId); +void vnodeBackup(int32_t vgId); +void vnodeGetStatus(struct SStatusMsg *status); + +SVnode *vnodeAcquire(int32_t vgId); +SVnode *vnodeAcquireNotClose(int32_t vgId); +void vnodeRelease(SVnode *pVnode); #ifdef __cplusplus } #endif -#endif +#endif /*_TD_VNODE_MAIN_H_*/ diff --git a/src/vnode/inc/vnodeBackup.h b/source/server/vnode/inc/vnodeMgmt.h similarity index 80% rename from src/vnode/inc/vnodeBackup.h rename to source/server/vnode/inc/vnodeMgmt.h index 0a6b26546c809fd27701bf8194243a2a79561fcb..23dc826db740ed54fdf61ffe456fe289215272d8 100644 --- a/src/vnode/inc/vnodeBackup.h +++ b/source/server/vnode/inc/vnodeMgmt.h @@ -13,20 +13,20 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_BACKUP_H -#define TDENGINE_VNODE_BACKUP_H +#ifndef _TD_VNODE_MGMT_H_ +#define _TD_VNODE_MGMT_H_ #ifdef __cplusplus extern "C" { #endif #include "vnodeInt.h" -int32_t vnodeInitBackup(); -void vnodeCleanupBackup(); -int32_t vnodeBackup(int32_t vgId); +int32_t vnodeInitMgmt(); +void vnodeCleanupMgmt(); +void vnodeProcessMgmtMsg(SRpcMsg *pMsg); #ifdef __cplusplus } #endif -#endif +#endif /*_TD_VNODE_MGMT_H_*/ diff --git a/src/vnode/inc/vnodeRead.h b/source/server/vnode/inc/vnodeMgmtMsg.h similarity index 64% rename from src/vnode/inc/vnodeRead.h rename to source/server/vnode/inc/vnodeMgmtMsg.h index 0e9655f837e311fd8d7d8779ce915e5b152be288..4d5533f2fe0adc25228f616827e2ae74572adb81 100644 --- a/src/vnode/inc/vnodeRead.h +++ b/source/server/vnode/inc/vnodeMgmtMsg.h @@ -13,24 +13,23 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_READ_H -#define TDENGINE_VNODE_READ_H +#ifndef _TD_VNODE_MGMT_MSG_H_ +#define _TD_VNODE_MGMT_MSG_H_ #ifdef __cplusplus extern "C" { #endif #include "vnodeInt.h" -int32_t vnodeInitRead(void); -void vnodeCleanupRead(void); - -int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam); -void vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead); -int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); -void vnodeWaitReadCompleted(SVnodeObj *pVnode); +int32_t vnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg); +int32_t vnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg); +int32_t vnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg); +int32_t vnodeProcessCompactVnodeMsg(SRpcMsg *rpcMsg); +int32_t vnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg); +int32_t vnodeProcessAlterStreamReq(SRpcMsg *pMsg); #ifdef __cplusplus } #endif -#endif +#endif /*_TD_VNODE_MGMT_H_*/ diff --git a/src/vnode/inc/vnodeMgmt.h b/source/server/vnode/inc/vnodeRead.h similarity index 55% rename from src/vnode/inc/vnodeMgmt.h rename to source/server/vnode/inc/vnodeRead.h index 5a7e7456195f2f8a21f2c99b4d50761c83729acd..e5efae3d9314775df128e39272b264e1aa773d9f 100644 --- a/src/vnode/inc/vnodeMgmt.h +++ b/source/server/vnode/inc/vnodeRead.h @@ -13,30 +13,30 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_MGMT_H -#define TDENGINE_VNODE_MGMT_H +#ifndef _TD_VNODE_READ_H_ +#define _TD_VNODE_READ_H_ #ifdef __cplusplus extern "C" { #endif #include "vnodeInt.h" -int32_t vnodeInitMgmt(); -void vnodeCleanupMgmt(); +int32_t vnodeInitRead(); +void vnodeCleanupRead(); +taos_queue vnodeAllocQueryQueue(SVnode *pVnode); +taos_queue vnodeAllocFetchQueue(SVnode *pVnode); +void vnodeFreeQueryQueue(taos_queue pQueue); +void vnodeFreeFetchQueue(taos_queue pQueue); -void* vnodeAcquire(int32_t vgId); -void vnodeRelease(void *pVnode); -void* vnodeGetWal(void *pVnode); +void vnodeProcessReadMsg(SRpcMsg *pRpcMsg); +int32_t vnodeReputPutToRQueue(SVnode *pVnode, void **qhandle, void *ahandle); -int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); -void vnodeBuildStatusMsg(void *pStatus); -void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); - -void vnodeAddIntoHash(SVnodeObj* pVnode); -void vnodeRemoveFromHash(SVnodeObj * pVnode); +void vnodeStartRead(SVnode *pVnode); +void vnodeStopRead(SVnode *pVnode); +void vnodeWaitReadCompleted(SVnode *pVnode); #ifdef __cplusplus } #endif -#endif +#endif /*_TD_VNODE_READ_H_*/ diff --git a/src/vnode/inc/vnodeWrite.h b/source/server/vnode/inc/vnodeReadMsg.h similarity index 61% rename from src/vnode/inc/vnodeWrite.h rename to source/server/vnode/inc/vnodeReadMsg.h index e996bc0b06f4563b24e5ebdfc90420b9e67df003..a1efb729e1779ce64f4209365f6d72ca96baf388 100644 --- a/src/vnode/inc/vnodeWrite.h +++ b/source/server/vnode/inc/vnodeReadMsg.h @@ -13,24 +13,32 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_WRITE_H -#define TDENGINE_VNODE_WRITE_H +#ifndef _TD_VNODE_READ_MSG_H_ +#define _TD_VNODE_READ_MSG_H_ #ifdef __cplusplus extern "C" { #endif #include "vnodeInt.h" -int32_t vnodeInitWrite(void); -void vnodeCleanupWrite(void); +typedef struct SReadMsg { + int32_t code; + int32_t contLen; + int8_t qtype; + int8_t msgType; + SVnode *pVnode; + SVnRsp rspRet; + void * rpcHandle; + void * rpcAhandle; + void * qhandle; + char pCont[]; +} SReadMsg; -int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg); -void vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite); -int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet); -void vnodeWaitWriteCompleted(SVnodeObj *pVnode); +int32_t vnodeProcessQueryMsg(SVnode *pVnode, SReadMsg *pRead); +int32_t vnodeProcessFetchMsg(SVnode *pVnode, SReadMsg *pRead); #ifdef __cplusplus } #endif -#endif \ No newline at end of file +#endif /*_TD_VNODE_READ_MSG_H_*/ diff --git a/src/vnode/inc/vnodeStatus.h b/source/server/vnode/inc/vnodeStatus.h similarity index 59% rename from src/vnode/inc/vnodeStatus.h rename to source/server/vnode/inc/vnodeStatus.h index 910a6d71b201fcdc9fbc1daa99fb57d0227d6093..c7f1b4c96d80475305c623f9d77194501e8d132f 100644 --- a/src/vnode/inc/vnodeStatus.h +++ b/source/server/vnode/inc/vnodeStatus.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_STATUS_H -#define TDENGINE_VNODE_STATUS_H +#ifndef _TD_VNODE_STATUS_H_ +#define _TD_VNODE_STATUS_H_ #ifdef __cplusplus extern "C" { @@ -25,24 +25,23 @@ typedef enum _VN_STATUS { TAOS_VN_STATUS_INIT = 0, TAOS_VN_STATUS_READY = 1, TAOS_VN_STATUS_CLOSING = 2, - TAOS_VN_STATUS_UPDATING = 3, - TAOS_VN_STATUS_RESET = 4, + TAOS_VN_STATUS_UPDATING = 3 } EVnodeStatus; -bool vnodeSetInitStatus(SVnodeObj* pVnode); -bool vnodeSetReadyStatus(SVnodeObj* pVnode); -bool vnodeSetClosingStatus(SVnodeObj* pVnode); -bool vnodeSetUpdatingStatus(SVnodeObj* pVnode); -bool vnodeSetResetStatus(SVnodeObj* pVnode); +// vnodeStatus +extern char* vnodeStatus[]; -bool vnodeInInitStatus(SVnodeObj* pVnode); -bool vnodeInReadyStatus(SVnodeObj* pVnode); -bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode); -bool vnodeInClosingStatus(SVnodeObj* pVnode); -bool vnodeInResetStatus(SVnodeObj* pVnode); +bool vnodeSetInitStatus(SVnode* pVnode); +bool vnodeSetReadyStatus(SVnode* pVnode); +bool vnodeSetClosingStatus(SVnode* pVnode); +bool vnodeSetUpdatingStatus(SVnode* pVnode); + +bool vnodeInInitStatus(SVnode* pVnode); +bool vnodeInReadyStatus(SVnode* pVnode); +bool vnodeInClosingStatus(SVnode* pVnode); #ifdef __cplusplus } #endif -#endif \ No newline at end of file +#endif /*_TD_VNODE_STATUS_H_*/ \ No newline at end of file diff --git a/src/vnode/inc/vnodeVersion.h b/source/server/vnode/inc/vnodeVersion.h similarity index 80% rename from src/vnode/inc/vnodeVersion.h rename to source/server/vnode/inc/vnodeVersion.h index 913e3915ab3b911746440be9eab2c7f2e05dce3d..81e6758559f8890afe1f013df75bf6b8263609a9 100644 --- a/src/vnode/inc/vnodeVersion.h +++ b/source/server/vnode/inc/vnodeVersion.h @@ -13,19 +13,19 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_VERSION_H -#define TDENGINE_VNODE_VERSION_H +#ifndef _TD_VNODE_VERSION_H_ +#define _TD_VNODE_VERSION_H_ #ifdef __cplusplus extern "C" { #endif #include "vnodeInt.h" -int32_t vnodeReadVersion(SVnodeObj *pVnode); -int32_t vnodeSaveVersion(SVnodeObj *pVnode); +int32_t vnodeReadVersion(SVnode *pVnode); +int32_t vnodeSaveVersion(SVnode *pVnode); #ifdef __cplusplus } #endif -#endif +#endif /*_TD_VNODE_VERSION_H_*/ diff --git a/src/vnode/inc/vnodeWorker.h b/source/server/vnode/inc/vnodeWorker.h similarity index 71% rename from src/vnode/inc/vnodeWorker.h rename to source/server/vnode/inc/vnodeWorker.h index 01d9d42900ef7e0c56bd396f5c698a43c4b501f8..eea35011a8ca0c5da4094cf7d522ae615fc57a9f 100644 --- a/src/vnode/inc/vnodeWorker.h +++ b/source/server/vnode/inc/vnodeWorker.h @@ -13,21 +13,22 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODE_WORKER_H -#define TDENGINE_VNODE_WORKER_H +#ifndef _TD_VNODE_WORKER_H_ +#define _TD_VNODE_WORKER_H_ #ifdef __cplusplus extern "C" { #endif #include "vnodeInt.h" -int32_t vnodeInitMWorker(); -void vnodeCleanupMWorker(); -int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode); -int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode); +int32_t vnodeInitWorker(); +void vnodeCleanupWorker(); +void vnodeProcessCleanupTask(SVnode *pVnode); +void vnodeProcessDestroyTask(SVnode *pVnode); +void vnodeProcessBackupTask(SVnode *pVnode); #ifdef __cplusplus } #endif -#endif \ No newline at end of file +#endif /*_TD_VNODE_WORKER_H_*/ \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeWrite.h b/source/server/vnode/inc/vnodeWrite.h index b4430cd2557a804f34594c118e3318bb971fdef7..48acf750c1911e39d934ea2886e7788f8f6bff6f 100644 --- a/source/server/vnode/inc/vnodeWrite.h +++ b/source/server/vnode/inc/vnodeWrite.h @@ -16,6 +16,25 @@ #ifndef _TD_VNODE_WRITE_H_ #define _TD_VNODE_WRITE_H_ -int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp); +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeInitWrite(); +void vnodeCleanupWrite(); +taos_queue vnodeAllocWriteQueue(SVnode *pVnode); +void vnodeFreeWriteQueue(taos_queue pQueue); + +void vnodeProcessWriteMsg(SRpcMsg *pRpcMsg); +int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead); + +void vnodeStartWrite(SVnode *pVnode); +void vnodeStopWrite(SVnode *pVnode); +void vnodeWaitWriteCompleted(SVnode *pVnode); + +#ifdef __cplusplus +} +#endif #endif /*_TD_VNODE_WRITE_H_*/ \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeWriteMsg.h b/source/server/vnode/inc/vnodeWriteMsg.h new file mode 100644 index 0000000000000000000000000000000000000000..86cdba694678b6c5feff7d08836c405ebf38faae --- /dev/null +++ b/source/server/vnode/inc/vnodeWriteMsg.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_WRITE_MSG_H_ +#define _TD_VNODE_WRITE_MSG_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp); +int32_t vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp); +int32_t vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp); +int32_t vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp); +int32_t vnodeProcessDropStableReq(SVnode *pVnode, SDropStableReq *pReq, SDropStableRsp *pRsp); +int32_t vnodeProcessUpdateTagValReq(SVnode *pVnode, SUpdateTagValReq *pReq, SUpdateTagValRsp *pRsp); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_WRITE_MSG_H_*/ \ No newline at end of file diff --git a/src/vnode/src/vnodeCfg.c b/source/server/vnode/src/vnodeCfg.c similarity index 97% rename from src/vnode/src/vnodeCfg.c rename to source/server/vnode/src/vnodeCfg.c index 2e1d761fcf8d3f51a2dff067a6020f125a3df5c9..9c01a47f8f23f08b04b9557b566f4501d587ccf8 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/source/server/vnode/src/vnodeCfg.c @@ -17,10 +17,10 @@ #include "os.h" #include "cJSON.h" #include "tglobal.h" -#include "dnode.h" #include "vnodeCfg.h" -static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { +static void vnodeLoadCfg(SVnode *pVnode, SCreateVnodeMsg *vnodeMsg) { +#if 0 tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db)); pVnode->dbCfgVersion = vnodeMsg->cfg.dbCfgVersion; pVnode->vgCfgVersion = vnodeMsg->cfg.vgCfgVersion; @@ -56,9 +56,11 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { SNodeInfo *node = &pVnode->syncCfg.nodeInfo[i]; vInfo("vgId:%d, dnode:%d, %s:%u", pVnode->vgId, node->nodeId, node->nodeFqdn, node->nodePort); } +#endif } -int32_t vnodeReadCfg(SVnodeObj *pVnode) { +int32_t vnodeReadCfg(SVnode *pVnode) { +#if 0 int32_t ret = TSDB_CODE_VND_APP_ERROR; int32_t len = 0; int maxLen = 1000; @@ -66,6 +68,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { cJSON * root = NULL; FILE * fp = NULL; bool nodeChanged = false; + SCreateVnodeMsg vnodeMsg; char file[TSDB_FILENAME_LEN + 30] = {0}; @@ -286,8 +289,13 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN); - bool changed = dnodeCheckEpChanged(node->nodeId, node->nodeEp); - if (changed) nodeChanged = changed; + char nodeEpStr[TSDB_EP_LEN]; + vnodeGetDnodeEp(node->nodeId, nodeEpStr, NULL, NULL); + bool changed = (strcmp(node->nodeEp, nodeEpStr) != 0); + if (changed) { + tstrncpy(node->nodeEp, nodeEpStr, TSDB_EP_LEN); + nodeChanged = changed; + } } ret = TSDB_CODE_SUCCESS; @@ -350,7 +358,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) { SVnodeDesc *node = &pMsg->nodes[i]; - dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL); + vnodeGetDnodeEp(node->nodeId, node->nodeEp, NULL, NULL); len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", node->nodeId); len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", node->nodeEp); if (i < pMsg->cfg.vgReplica - 1) { @@ -368,5 +376,6 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { terrno = 0; vInfo("vgId:%d, successed to write %s", pMsg->cfg.vgId, file); +#endif return TSDB_CODE_SUCCESS; } diff --git a/source/server/vnode/src/vnodeInt.c b/source/server/vnode/src/vnodeInt.c index ac14d2a75643432c4df2584cf6416cafb1ee223c..a10c35fd987575635e0aa8988c21b02ac8448683 100644 --- a/source/server/vnode/src/vnodeInt.c +++ b/source/server/vnode/src/vnodeInt.c @@ -13,16 +13,39 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#define _DEFAULT_SOURCE +#include "os.h" +#include "tstep.h" +#include "vnodeMain.h" +#include "vnodeMgmt.h" +#include "vnodeRead.h" +#include "vnodeWorker.h" +#include "vnodeWrite.h" -int32_t vnodeInit(SVnodePara para) { return 0; } +static struct { + struct SSteps *steps; + SVnodeFp fp; +} tsVint; -void vnodeCleanup() {} +int32_t vnodeInit(SVnodePara para) { + tsVint.fp = para.fp; -int32_t vnodeGetStatistics(SVnodeStat *stat) { return 0; } + struct SSteps *steps = taosStepInit(8, NULL); + if (steps == NULL) return -1; -void vnodeGetStatus(struct SStatusMsg *status) {} + taosStepAdd(steps, "vnode-main", vnodeInitMain, vnodeCleanupMain); + taosStepAdd(steps, "vnode-worker",vnodeInitWorker, vnodeCleanupWorker); + taosStepAdd(steps, "vnode-read", vnodeInitRead, vnodeCleanupRead); + taosStepAdd(steps, "vnode-mgmt", vnodeInitMgmt, vnodeCleanupMgmt); + taosStepAdd(steps, "vnode-write", vnodeInitWrite, vnodeCleanupWrite); + // taosStepAdd(steps, "vnode-queue", tsdbInitCommitQueue, tsdbDestroyCommitQueue); -void vnodeSetAccess(struct SVgroupAccess *access, int32_t numOfVnodes) {} + tsVint.steps = steps; + return taosStepExec(tsVint.steps); +} -void vnodeProcessMsg(SRpcMsg *msg) {} +void vnodeCleanup() { taosStepCleanup(tsVint.steps); } + +void vnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port) { + return (*tsVint.fp.GetDnodeEp)(dnodeId, ep, fqdn, port); +} \ No newline at end of file diff --git a/src/vnode/src/vnodeMain.c b/source/server/vnode/src/vnodeMain.c similarity index 52% rename from src/vnode/src/vnodeMain.c rename to source/server/vnode/src/vnodeMain.c index c823880ae2028c4bcfe26dbfc5cd60af62443722..d9c1a88d157aff2741912011e1f67e149a4a7f20 100644 --- a/src/vnode/src/vnodeMain.c +++ b/source/server/vnode/src/vnodeMain.c @@ -18,30 +18,104 @@ #include "taoserror.h" #include "taosmsg.h" #include "tglobal.h" -#include "tfs.h" -#include "query.h" -#include "dnode.h" +#include "ttimer.h" +#include "thash.h" +// #include "query.h" #include "vnodeCfg.h" +#include "vnodeMain.h" +#include "vnodeMgmt.h" +#include "vnodeRead.h" #include "vnodeStatus.h" -#include "vnodeSync.h" #include "vnodeVersion.h" -#include "vnodeMgmt.h" #include "vnodeWorker.h" -#include "vnodeBackup.h" -#include "vnodeMain.h" +#include "vnodeWrite.h" + +typedef struct { + pthread_t thread; + int32_t threadIndex; + int32_t failed; + int32_t opened; + int32_t vnodeNum; + int32_t * vnodeList; +} SOpenVnodeThread; + +static struct { + void * timer; + SHashObj *hash; + int32_t openVnodes; + int32_t totalVnodes; + void (*msgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); +} tsVmain; + +static void vnodeIncRef(void *ptNode) { + assert(ptNode != NULL); + + SVnode **ppVnode = (SVnode **)ptNode; + assert(ppVnode); + assert(*ppVnode); + + SVnode *pVnode = *ppVnode; + atomic_add_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); +} + +SVnode *vnodeAcquire(int32_t vgId) { + SVnode *pVnode = NULL; + +#if 0 + taosHashGetClone(tsVmain.hash, &vgId, sizeof(int32_t), vnodeIncRef, &pVnode); +#endif + if (pVnode == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + vDebug("vgId:%d, not exist", vgId); + return NULL; + } + + return pVnode; +} + +SVnode *vnodeAcquireNotClose(int32_t vgId) { + SVnode *pVnode = vnodeAcquire(vgId); + if (pVnode != NULL && pVnode->preClose == 1) { + vnodeRelease(pVnode); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + vDebug("vgId:%d, not exist, pre closing", vgId); + return NULL; + } + + return pVnode; +} + +void vnodeRelease(SVnode *pVnode) { + if (pVnode == NULL) return; + + int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + int32_t vgId = pVnode->vgId; + + vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", vgId, refCount, pVnode); + assert(refCount >= 0); + + if (refCount <= 0) { + vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", vgId, refCount, pVnode); + vnodeProcessDestroyTask(pVnode); + int32_t count = taosHashGetSize(tsVmain.hash); + vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count); + } +} static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { int32_t code; - SVnodeObj *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId); + SVnode *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId); if (pVnode != NULL) { vDebug("vgId:%d, vnode already exist, refCount:%d pVnode:%p", pVnodeCfg->cfg.vgId, pVnode->refCount, pVnode); vnodeRelease(pVnode); return TSDB_CODE_SUCCESS; } +#if 0 if (tfsMkdir("vnode") < 0) { vError("vgId:%d, failed to create vnode dir, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); return terrno; @@ -64,7 +138,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); return TSDB_CODE_VND_INIT_FAILED; } - +#endif vInfo("vgId:%d, vnode dir is created, walLevel:%d fsyncPeriod:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel, pVnodeCfg->cfg.fsyncPeriod); code = vnodeOpen(pVnodeCfg->cfg.vgId); @@ -73,7 +147,8 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { } int32_t vnodeSync(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); +#if 0 + SVnode *pVnode = vnodeAcquireNotClose(vgId); if (pVnode == NULL) { vDebug("vgId:%d, failed to sync, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; @@ -90,13 +165,12 @@ int32_t vnodeSync(int32_t vgId) { } vnodeRelease(pVnode); - +#endif return TSDB_CODE_SUCCESS; } - int32_t vnodeDrop(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); + SVnode *pVnode = vnodeAcquireNotClose(vgId); if (pVnode == NULL) { vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; @@ -110,25 +184,29 @@ int32_t vnodeDrop(int32_t vgId) { pVnode->dropped = 1; vnodeRelease(pVnode); - vnodeCleanupInMWorker(pVnode); + vnodeProcessCleanupTask(pVnode); return TSDB_CODE_SUCCESS; } + int32_t vnodeCompact(int32_t vgId) { - void *pVnode = vnodeAcquire(vgId); +#if 0 + SVnode *pVnode = vnodeAcquire(vgId); if (pVnode != NULL) { vDebug("vgId:%d, compact vnode msg is received", vgId); - //not care success or not - tsdbCompact(((SVnodeObj*)pVnode)->tsdb); + // not care success or not + tsdbCompact(((SVnode *)pVnode)->tsdb); vnodeRelease(pVnode); } else { vInfo("vgId:%d, vnode not exist, can't compact it", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } - return TSDB_CODE_SUCCESS; +#endif + return TSDB_CODE_SUCCESS; } -static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) { +static int32_t vnodeAlterImp(SVnode *pVnode, SCreateVnodeMsg *pVnodeCfg) { +#if 0 STsdbCfg tsdbCfg = pVnode->tsdbCfg; SSyncCfg syncCfg = pVnode->syncCfg; int32_t dbCfgVersion = pVnode->dbCfgVersion; @@ -204,13 +282,11 @@ static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) { vnodeSetReadyStatus(pVnode); } - +#endif return 0; } -int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) { - SVnodeObj *pVnode = vparam; - +int32_t vnodeAlter(SVnode *pVnode, SCreateVnodeMsg *pVnodeCfg) { vDebug("vgId:%d, current dbCfgVersion:%d vgCfgVersion:%d, input dbCfgVersion:%d vgCfgVersion:%d", pVnode->vgId, pVnode->dbCfgVersion, pVnode->vgCfgVersion, pVnodeCfg->cfg.dbCfgVersion, pVnodeCfg->cfg.vgCfgVersion); @@ -231,6 +307,7 @@ int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) { } static void vnodeFindWalRootDir(int32_t vgId, char *walRootDir) { +#if 0 char vnodeDir[TSDB_FILENAME_LEN] = "\0"; snprintf(vnodeDir, TSDB_FILENAME_LEN, "/vnode/vnode%d/wal", vgId); @@ -246,15 +323,17 @@ static void vnodeFindWalRootDir(int32_t vgId, char *walRootDir) { sprintf(walRootDir, "%s/vnode/vnode%d", TFS_DISK_PATH(tfile->level, tfile->id), vgId); tfsClosedir(tdir); +#endif } int32_t vnodeOpen(int32_t vgId) { +#if 0 char temp[TSDB_FILENAME_LEN * 3]; char rootDir[TSDB_FILENAME_LEN * 2]; char walRootDir[TSDB_FILENAME_LEN * 2] = {0}; snprintf(rootDir, TSDB_FILENAME_LEN * 2, "%s/vnode%d", tsVnodeDir, vgId); - SVnodeObj *pVnode = calloc(sizeof(SVnodeObj), 1); + SVnode *pVnode = calloc(sizeof(SVnode), 1); if (pVnode == NULL) { vError("vgId:%d, failed to open vnode since no enough memory", vgId); return TAOS_SYSTEM_ERROR(errno); @@ -262,9 +341,9 @@ int32_t vnodeOpen(int32_t vgId) { atomic_add_fetch_32(&pVnode->refCount, 1); - pVnode->vgId = vgId; + pVnode->vgId = vgId; pVnode->fversion = 0; - pVnode->version = 0; + pVnode->version = 0; pVnode->tsdbCfg.tsdbId = pVnode->vgId; pVnode->rootDir = strdup(rootDir); pVnode->accessState = TSDB_VN_ALL_ACCCESS; @@ -279,7 +358,7 @@ int32_t vnodeOpen(int32_t vgId) { vError("vgId:%d, failed to read config file, set cfgVersion to 0", pVnode->vgId); vnodeCleanUp(pVnode); return 0; - } + } code = vnodeReadVersion(pVnode); if (code != TSDB_CODE_SUCCESS) { @@ -291,29 +370,15 @@ int32_t vnodeOpen(int32_t vgId) { } pVnode->fversion = pVnode->version; - - pVnode->wqueue = dnodeAllocVWriteQueue(pVnode); - pVnode->qqueue = dnodeAllocVQueryQueue(pVnode); - pVnode->fqueue = dnodeAllocVFetchQueue(pVnode); + + pVnode->wqueue = vnodeAllocWriteQueue(pVnode); + pVnode->qqueue = vnodeAllocQueryQueue(pVnode); + pVnode->fqueue = vnodeAllocFetchQueue(pVnode); if (pVnode->wqueue == NULL || pVnode->qqueue == NULL || pVnode->fqueue == NULL) { vnodeCleanUp(pVnode); return terrno; } - if (tsEnableStream) { - SCqCfg cqCfg = {0}; - sprintf(cqCfg.user, "_root"); - strcpy(cqCfg.pass, tsInternalPass); - strcpy(cqCfg.db, pVnode->db); - cqCfg.vgId = vgId; - cqCfg.cqWrite = vnodeWriteToCache; - pVnode->cq = cqOpen(pVnode, &cqCfg); - if (pVnode->cq == NULL) { - vnodeCleanUp(pVnode); - return terrno; - } - } - STsdbAppH appH = {0}; appH.appH = (void *)pVnode; appH.notifyStatus = vnodeProcessTsdbStatus; @@ -327,8 +392,8 @@ int32_t vnodeOpen(int32_t vgId) { vnodeCleanUp(pVnode); return terrno; } else if (tsdbGetState(pVnode->tsdb) != TSDB_STATE_OK) { - vError("vgId:%d, failed to open tsdb(state: %d), replica:%d reason:%s", pVnode->vgId, - tsdbGetState(pVnode->tsdb), pVnode->syncCfg.replica, tstrerror(terrno)); + vError("vgId:%d, failed to open tsdb(state: %d), replica:%d reason:%s", pVnode->vgId, tsdbGetState(pVnode->tsdb), + pVnode->syncCfg.replica, tstrerror(terrno)); if (pVnode->syncCfg.replica <= 1) { vnodeCleanUp(pVnode); return TSDB_CODE_VND_INVALID_TSDB_STATE; @@ -355,12 +420,12 @@ int32_t vnodeOpen(int32_t vgId) { sprintf(temp, "%s/wal", walRootDir); pVnode->walCfg.vgId = pVnode->vgId; pVnode->wal = walOpen(temp, &pVnode->walCfg); - if (pVnode->wal == NULL) { + if (pVnode->wal == NULL) { vnodeCleanUp(pVnode); return terrno; } - walRestore(pVnode->wal, pVnode, vnodeProcessWrite); + walRestore(pVnode->wal, pVnode, (FWalWrite)vnodeProcessWalMsg); if (pVnode->version == 0) { pVnode->fversion = 0; pVnode->version = walGetVersion(pVnode->wal); @@ -386,40 +451,16 @@ int32_t vnodeOpen(int32_t vgId) { vDebug("vgId:%d, vnode is opened in %s - %s, pVnode:%p", pVnode->vgId, rootDir, walRootDir, pVnode); - vnodeAddIntoHash(pVnode); - - SSyncInfo syncInfo; - syncInfo.vgId = pVnode->vgId; - syncInfo.version = pVnode->version; - syncInfo.syncCfg = pVnode->syncCfg; - tstrncpy(syncInfo.path, walRootDir, TSDB_FILENAME_LEN); - syncInfo.getWalInfoFp = vnodeGetWalInfo; - syncInfo.writeToCacheFp = vnodeWriteToCache; - syncInfo.confirmForward = vnodeConfirmForard; - syncInfo.notifyRoleFp = vnodeNotifyRole; - syncInfo.notifyFlowCtrlFp = vnodeCtrlFlow; - syncInfo.startSyncFileFp = vnodeStartSyncFile; - syncInfo.stopSyncFileFp = vnodeStopSyncFile; - syncInfo.getVersionFp = vnodeGetVersion; - syncInfo.sendFileFp = tsdbSyncSend; - syncInfo.recvFileFp = tsdbSyncRecv; - syncInfo.pTsdb = pVnode->tsdb; - pVnode->sync = syncStart(&syncInfo); - - if (pVnode->sync <= 0) { - vError("vgId:%d, failed to open sync, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, - tstrerror(terrno)); - vnodeRemoveFromHash(pVnode); - vnodeCleanUp(pVnode); - return terrno; - } + taosHashPut(tsVmain.hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnode *)); vnodeSetReadyStatus(pVnode); + pVnode->role = TAOS_SYNC_ROLE_MASTER; +#endif return TSDB_CODE_SUCCESS; } int32_t vnodeClose(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); + SVnode *pVnode = vnodeAcquireNotClose(vgId); if (pVnode == NULL) return 0; if (pVnode->dropped) { vnodeRelease(pVnode); @@ -435,10 +476,11 @@ int32_t vnodeClose(int32_t vgId) { return 0; } -void vnodeDestroy(SVnodeObj *pVnode) { +void vnodeDestroy(SVnode *pVnode) { +#if 0 int32_t code = 0; int32_t vgId = pVnode->vgId; - + if (pVnode->qMgmt) { qCleanupQueryMgmt(pVnode->qMgmt); pVnode->qMgmt = NULL; @@ -475,47 +517,49 @@ void vnodeDestroy(SVnodeObj *pVnode) { } if (pVnode->wqueue) { - dnodeFreeVWriteQueue(pVnode->wqueue); + vnodeFreeWriteQueue(pVnode->wqueue); pVnode->wqueue = NULL; } if (pVnode->qqueue) { - dnodeFreeVQueryQueue(pVnode->qqueue); + vnodeFreeQueryQueue(pVnode->qqueue); pVnode->qqueue = NULL; } if (pVnode->fqueue) { - dnodeFreeVFetchQueue(pVnode->fqueue); + vnodeFreeFetchQueue(pVnode->fqueue); pVnode->fqueue = NULL; } tfree(pVnode->rootDir); if (pVnode->dropped) { - char rootDir[TSDB_FILENAME_LEN] = {0}; + char rootDir[TSDB_FILENAME_LEN] = {0}; char stagingDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", "vnode", vgId); sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); tfsRename(rootDir, stagingDir); - vnodeBackup(vgId); + vnodeProcessBackupTask(pVnode); - dnodeSendStatusMsgToMnode(); + // dnodeSendStatusMsgToMnode(); } tsem_destroy(&pVnode->sem); pthread_mutex_destroy(&pVnode->statusMutex); free(pVnode); tsdbDecCommitRef(vgId); +#endif } -void vnodeCleanUp(SVnodeObj *pVnode) { +void vnodeCleanUp(SVnode *pVnode) { +#if 0 vDebug("vgId:%d, vnode will cleanup, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); vnodeSetClosingStatus(pVnode); - vnodeRemoveFromHash(pVnode); + taosHashRemove(tsVmain.hash, &pVnode->vgId, sizeof(int32_t)); // stop replication module if (pVnode->sync > 0) { @@ -526,10 +570,12 @@ void vnodeCleanUp(SVnodeObj *pVnode) { vDebug("vgId:%d, vnode is cleaned, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); vnodeRelease(pVnode); +#endif } +#if 0 static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { - SVnodeObj *pVnode = arg; + SVnode *pVnode = arg; if (eno != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to commit since %s, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, tstrerror(eno), @@ -561,9 +607,301 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { } // timer thread callback - if(status == TSDB_STATUS_COMMIT_NOBLOCK) { + if (status == TSDB_STATUS_COMMIT_NOBLOCK) { qSolveCommitNoBlock(pVnode->tsdb, pVnode->qMgmt); } return 0; } +#endif + +static void *vnodeOpenVnode(void *param) { + SOpenVnodeThread *pThread = param; + + vDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + setThreadName("vnodeOpenVnode"); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + int32_t vgId = pThread->vnodeList[v]; + + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", vgId, + tsVmain.openVnodes, tsVmain.totalVnodes); + // (*vnodeInst()->fp.ReportStartup)("open-vnodes", stepDesc); + + if (vnodeOpen(vgId) < 0) { + vError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex); + pThread->failed++; + } else { + vDebug("vgId:%d, is opened by thread:%d", vgId, pThread->threadIndex); + pThread->opened++; + } + + atomic_add_fetch_32(&tsVmain.openVnodes, 1); + } + + vDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t vnodeGetVnodeListFromDisk(int32_t vnodeList[], int32_t *numOfVnodes) { +#if 0 + DIR *dir = opendir(tsVnodeDir); + if (dir == NULL) return TSDB_CODE_DND_NO_WRITE_ACCESS; + + *numOfVnodes = 0; + struct dirent *de = NULL; + while ((de = readdir(dir)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + if (de->d_type & DT_DIR) { + if (strncmp("vnode", de->d_name, 5) != 0) continue; + int32_t vnode = atoi(de->d_name + 5); + if (vnode == 0) continue; + + (*numOfVnodes)++; + + if (*numOfVnodes >= TSDB_MAX_VNODES) { + vError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES); + closedir(dir); + return TSDB_CODE_DND_TOO_MANY_VNODES; + } else { + vnodeList[*numOfVnodes - 1] = vnode; + } + } + } + closedir(dir); +#endif + return TSDB_CODE_SUCCESS; +} + +static int32_t vnodeOpenVnodes() { + int32_t vnodeList[TSDB_MAX_VNODES] = {0}; + int32_t numOfVnodes = 0; + int32_t status = vnodeGetVnodeListFromDisk(vnodeList, &numOfVnodes); + + if (status != TSDB_CODE_SUCCESS) { + vInfo("failed to get vnode list from disk since code:%d", status); + return status; + } + + tsVmain.totalVnodes = numOfVnodes; + + int32_t threadNum = tsNumOfCores; + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + + SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SOpenVnodeThread *pThread = &threads[t]; + pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v]; + } + + vInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); + + for (int32_t t = 0; t < threadNum; ++t) { + SOpenVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (pthread_create(&pThread->thread, &thAttr, vnodeOpenVnode, pThread) != 0) { + vError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + } + + int32_t openVnodes = 0; + int32_t failedVnodes = 0; + for (int32_t t = 0; t < threadNum; ++t) { + SOpenVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + pthread_join(pThread->thread, NULL); + } + openVnodes += pThread->opened; + failedVnodes += pThread->failed; + free(pThread->vnodeList); + } + + free(threads); + vInfo("there are total vnodes:%d, opened:%d", numOfVnodes, openVnodes); + + if (failedVnodes != 0) { + vError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes); + return -1; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { + void *pIter = taosHashIterate(tsVmain.hash, NULL); + while (pIter) { + SVnode **pVnode = pIter; + if (*pVnode) { + (*numOfVnodes)++; + if (*numOfVnodes >= TSDB_MAX_VNODES) { + vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES); + continue; + } else { + vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId; + } + } + + pIter = taosHashIterate(tsVmain.hash, pIter); + } + + return TSDB_CODE_SUCCESS; +} + +static void vnodeCleanupVnodes() { + int32_t vnodeList[TSDB_MAX_VNODES] = {0}; + int32_t numOfVnodes = 0; + + int32_t code = vnodeGetVnodeList(vnodeList, &numOfVnodes); + + if (code != TSDB_CODE_SUCCESS) { + vInfo("failed to get dnode list since code %d", code); + return; + } + + for (int32_t i = 0; i < numOfVnodes; ++i) { + vnodeClose(vnodeList[i]); + } + + vInfo("total vnodes:%d are all closed", numOfVnodes); +} + +static void vnodeInitMsgFp() { + tsVmain.msgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = vnodeProcessMgmtMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = vnodeProcessMgmtMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = vnodeProcessMgmtMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMgmtMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = vnodeProcessMgmtMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = vnodeProcessMgmtMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessWriteMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = vnodeProcessWriteMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = vnodeProcessWriteMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessWriteMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessWriteMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessWriteMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessReadMsg; + tsVmain.msgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessReadMsg; +} + +void vnodeProcessMsg(SRpcMsg *pMsg) { + if (tsVmain.msgFp[pMsg->msgType]) { + (*tsVmain.msgFp[pMsg->msgType])(pMsg); + } else { + assert(0); + } +} + +int32_t vnodeInitMain() { + vnodeInitMsgFp(); + + tsVmain.timer = taosTmrInit(100, 200, 60000, "VND-TIMER"); + if (tsVmain.timer == NULL) { + vError("failed to init vnode timer"); + return -1; + } + + tsVmain.hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (tsVmain.hash == NULL) { + taosTmrCleanUp(tsVmain.timer); + vError("failed to init vnode mgmt"); + return -1; + } + + vInfo("vnode main is initialized"); + return vnodeOpenVnodes(); +} + +void vnodeCleanupMain() { + taosTmrCleanUp(tsVmain.timer); + tsVmain.timer = NULL; + + vnodeCleanupVnodes(); + + taosHashCleanup(tsVmain.hash); + tsVmain.hash = NULL; +} + +static void vnodeBuildVloadMsg(SVnode *pVnode, SStatusMsg *pStatus) { +#if 0 + int64_t totalStorage = 0; + int64_t compStorage = 0; + int64_t pointsWritten = 0; + + if (vnodeInClosingStatus(pVnode)) return; + if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; + + if (pVnode->tsdb) { + tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage); + } + + SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; + pLoad->vgId = htonl(pVnode->vgId); + pLoad->dbCfgVersion = htonl(pVnode->dbCfgVersion); + pLoad->vgCfgVersion = htonl(pVnode->vgCfgVersion); + pLoad->totalStorage = htobe64(totalStorage); + pLoad->compStorage = htobe64(compStorage); + pLoad->pointsWritten = htobe64(pointsWritten); + pLoad->vnodeVersion = htobe64(pVnode->version); + pLoad->status = pVnode->status; + pLoad->role = pVnode->role; + pLoad->replica = pVnode->syncCfg.replica; + pLoad->compact = (pVnode->tsdb != NULL) ? tsdbGetCompactState(pVnode->tsdb) : 0; +#endif +} + +void vnodeGetStatus(struct SStatusMsg *pStatus) { + void *pIter = taosHashIterate(tsVmain.hash, NULL); + while (pIter) { + SVnode **pVnode = pIter; + if (*pVnode) { + vnodeBuildVloadMsg(*pVnode, pStatus); + } + pIter = taosHashIterate(tsVmain.hash, pIter); + } +} + +void vnodeSetAccess(struct SVgroupAccess *pAccess, int32_t numOfVnodes) { + for (int32_t i = 0; i < numOfVnodes; ++i) { + pAccess[i].vgId = htonl(pAccess[i].vgId); + SVnode *pVnode = vnodeAcquireNotClose(pAccess[i].vgId); + if (pVnode != NULL) { + pVnode->accessState = pAccess[i].accessState; + if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) { + vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState); + } + vnodeRelease(pVnode); + } + } +} + +void vnodeBackup(int32_t vgId) { + char newDir[TSDB_FILENAME_LEN] = {0}; + char stagingDir[TSDB_FILENAME_LEN] = {0}; + + sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); + sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); + +#if 0 + if (tsEnableVnodeBak) { + tfsRmdir(newDir); + tfsRename(stagingDir, newDir); + } else { + vInfo("vgId:%d, vnode backup not enabled", vgId); + + tfsRmdir(stagingDir); + } +#endif +} diff --git a/source/server/vnode/src/vnodeMgmt.c b/source/server/vnode/src/vnodeMgmt.c new file mode 100644 index 0000000000000000000000000000000000000000..4158b0f6aa96d30d48947007d01c73a849255606 --- /dev/null +++ b/source/server/vnode/src/vnodeMgmt.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "vnodeMain.h" +#include "vnodeMgmt.h" +#include "vnodeMgmtMsg.h" + +typedef struct { + SRpcMsg rpcMsg; + char pCont[]; +} SVnMgmtMsg; + +static struct { + SWorkerPool pool; + taos_queue pQueue; + int32_t (*msgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); +} tsVmgmt = {0}; + +static int32_t vnodeProcessMgmtStart(void *unused, SVnMgmtMsg *pMgmt, int32_t qtype) { + SRpcMsg *pMsg = &pMgmt->rpcMsg; + int32_t msgType = pMsg->msgType; + + if (tsVmgmt.msgFp[msgType]) { + vTrace("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[msgType]); + return (*tsVmgmt.msgFp[msgType])(pMsg); + } else { + vError("msg:%p, ahandle:%p type:%s not processed since no handle", pMgmt, pMsg->ahandle, taosMsg[msgType]); + return TSDB_CODE_DND_MSG_NOT_PROCESSED; + } +} + +static void vnodeSendMgmtEnd(void *unused, SVnMgmtMsg *pMgmt, int32_t qtype, int32_t code) { + SRpcMsg *pMsg = &pMgmt->rpcMsg; + SRpcMsg rsp = {0}; + + rsp.code = code; + vTrace("msg:%p, is processed, code:0x%x", pMgmt, rsp.code); + if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) { + rsp.handle = pMsg->handle; + rsp.pCont = NULL; + rpcSendResponse(&rsp); + } + + taosFreeQitem(pMsg); +} + +static void vnodeInitMgmtReqFp() { + tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = vnodeProcessCreateVnodeMsg; + tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = vnodeProcessAlterVnodeMsg; + tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = vnodeProcessSyncVnodeMsg; + tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE]= vnodeProcessCompactVnodeMsg; + tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = vnodeProcessDropVnodeMsg; + tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = vnodeProcessAlterStreamReq; +} + +static int32_t vnodeWriteToMgmtQueue(SRpcMsg *pMsg) { + int32_t size = sizeof(SVnMgmtMsg) + pMsg->contLen; + SVnMgmtMsg *pMgmt = taosAllocateQitem(size); + if (pMgmt == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; + + pMgmt->rpcMsg = *pMsg; + pMgmt->rpcMsg.pCont = pMgmt->pCont; + memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen); + taosWriteQitem(tsVmgmt.pQueue, TAOS_QTYPE_RPC, pMgmt); + + return TSDB_CODE_SUCCESS; +} + +void vnodeProcessMgmtMsg(SRpcMsg *pMsg) { + int32_t code = vnodeWriteToMgmtQueue(pMsg); + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = code}; + rpcSendResponse(&rsp); + } + + rpcFreeCont(pMsg->pCont); +} + +int32_t vnodeInitMgmt() { + vnodeInitMgmtReqFp(); + + SWorkerPool *pPool = &tsVmgmt.pool; + pPool->name = "vmgmt"; + pPool->startFp = (ProcessStartFp)vnodeProcessMgmtStart; + pPool->endFp = (ProcessEndFp)vnodeSendMgmtEnd; + pPool->min = 1; + pPool->max = 1; + if (tWorkerInit(pPool) != 0) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + tsVmgmt.pQueue = tWorkerAllocQueue(pPool, NULL); + + vInfo("vmgmt is initialized, max worker %d", pPool->max); + return TSDB_CODE_SUCCESS; +} + +void vnodeCleanupMgmt() { + tWorkerFreeQueue(&tsVmgmt.pool, tsVmgmt.pQueue); + tWorkerCleanup(&tsVmgmt.pool); + tsVmgmt.pQueue = NULL; + vInfo("vmgmt is closed"); +} diff --git a/source/server/vnode/src/vnodeMgmtMsg.c b/source/server/vnode/src/vnodeMgmtMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..d67fa11ece08bfb763d2bb11e59e3388e65b3ea6 --- /dev/null +++ b/source/server/vnode/src/vnodeMgmtMsg.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "vnodeMain.h" +#include "vnodeMgmtMsg.h" + +static SCreateVnodeMsg* vnodeParseVnodeMsg(SRpcMsg *rpcMsg) { + SCreateVnodeMsg *pCreate = rpcMsg->pCont; + pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); + pCreate->cfg.dbCfgVersion = htonl(pCreate->cfg.dbCfgVersion); + pCreate->cfg.vgCfgVersion = htonl(pCreate->cfg.vgCfgVersion); + pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); + pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); + pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); + pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); + pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1); + pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2); + pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); + pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); + pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); + pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod); + pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); + + for (int32_t j = 0; j < pCreate->cfg.vgReplica; ++j) { + pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId); + } + + return pCreate; +} + +int32_t vnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { + SCreateVnodeMsg *pCreate = vnodeParseVnodeMsg(rpcMsg); + SVnode *pVnode = vnodeAcquire(pCreate->cfg.vgId); + if (pVnode != NULL) { + vDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId); + vnodeRelease(pVnode); + return TSDB_CODE_SUCCESS; + } else { + vDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId); + return vnodeCreate(pCreate); + } +} + +int32_t vnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { + SAlterVnodeMsg *pAlter = vnodeParseVnodeMsg(rpcMsg); + + void *pVnode = vnodeAcquireNotClose(pAlter->cfg.vgId); + if (pVnode != NULL) { + vDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId); + int32_t code = vnodeAlter(pVnode, pAlter); + vnodeRelease(pVnode); + return code; + } else { + vInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } +} + +int32_t vnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg) { + SSyncVnodeMsg *pSyncVnode = rpcMsg->pCont; + pSyncVnode->vgId = htonl(pSyncVnode->vgId); + + return vnodeSync(pSyncVnode->vgId); +} + +int32_t vnodeProcessCompactVnodeMsg(SRpcMsg *rpcMsg) { + SCompactVnodeMsg *pCompactVnode = rpcMsg->pCont; + pCompactVnode->vgId = htonl(pCompactVnode->vgId); + return vnodeCompact(pCompactVnode->vgId); +} + +int32_t vnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { + SDropVnodeMsg *pDrop = rpcMsg->pCont; + pDrop->vgId = htonl(pDrop->vgId); + + return vnodeDrop(pDrop->vgId); +} + +int32_t vnodeProcessAlterStreamReq(SRpcMsg *pMsg) { return 0; } diff --git a/source/server/vnode/src/vnodeRead.c b/source/server/vnode/src/vnodeRead.c new file mode 100644 index 0000000000000000000000000000000000000000..39b6983b7d259e4c74a293a20fa58a70e4904bfa --- /dev/null +++ b/source/server/vnode/src/vnodeRead.c @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosmsg.h" +#include "tglobal.h" +// #include "query.h" +#include "vnodeMain.h" +#include "vnodeRead.h" +#include "vnodeReadMsg.h" +#include "vnodeStatus.h" + +static struct { + SWorkerPool query; + SWorkerPool fetch; + int32_t (*msgFp[TSDB_MSG_TYPE_MAX])(SVnode *, struct SReadMsg *); +} tsVread = {0}; + +void vnodeStartRead(SVnode *pVnode) {} +void vnodeStopRead(SVnode *pVnode) {} + +void vnodeWaitReadCompleted(SVnode *pVnode) { + while (pVnode->queuedRMsg > 0) { + vTrace("vgId:%d, queued rmsg num:%d", pVnode->vgId, pVnode->queuedRMsg); + taosMsleep(10); + } +} + +static int32_t vnodeWriteToRQueue(SVnode *pVnode, void *pCont, int32_t contLen, int8_t qtype, SRpcMsg *pRpcMsg) { + if (pVnode->dropped) { + return TSDB_CODE_APP_NOT_READY; + } + +#if 0 + if (!((pVnode->role == TAOS_SYNC_ROLE_MASTER) || (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE))) { + return TSDB_CODE_APP_NOT_READY; + } +#endif + + if (!vnodeInReadyStatus(pVnode)) { + vDebug("vgId:%d, failed to write into vread queue, vnode status is %s", pVnode->vgId, vnodeStatus[pVnode->status]); + return TSDB_CODE_APP_NOT_READY; + } + + int32_t size = sizeof(SReadMsg) + contLen; + SReadMsg *pRead = taosAllocateQitem(size); + if (pRead == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + if (pRpcMsg != NULL) { + pRead->rpcHandle = pRpcMsg->handle; + pRead->rpcAhandle = pRpcMsg->ahandle; + pRead->msgType = pRpcMsg->msgType; + pRead->code = pRpcMsg->code; + } + + if (contLen != 0) { + pRead->contLen = contLen; + memcpy(pRead->pCont, pCont, contLen); + } else { + pRead->qhandle = pCont; + } + + pRead->qtype = qtype; + + atomic_add_fetch_32(&pVnode->refCount, 1); + atomic_add_fetch_32(&pVnode->queuedRMsg, 1); + + if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRead->msgType == TSDB_MSG_TYPE_FETCH) { + return taosWriteQitem(pVnode->fqueue, qtype, pRead); + } else { + return taosWriteQitem(pVnode->qqueue, qtype, pRead); + } +} + +static void vnodeFreeFromRQueue(SVnode *pVnode, SReadMsg *pRead) { + atomic_sub_fetch_32(&pVnode->queuedRMsg, 1); + + taosFreeQitem(pRead); + vnodeRelease(pVnode); +} + +int32_t vnodeReputPutToRQueue(SVnode *pVnode, void **qhandle, void *ahandle) { + SRpcMsg rpcMsg = {0}; + rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; + rpcMsg.ahandle = ahandle; + + int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg); + if (code == TSDB_CODE_SUCCESS) { + vTrace("QInfo:%p add to vread queue for exec query", *qhandle); + } + + return code; +} + +void vnodeProcessReadMsg(SRpcMsg *pMsg) { + int32_t queuedMsgNum = 0; + int32_t leftLen = pMsg->contLen; + int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID; + char * pCont = pMsg->pCont; + + while (leftLen > 0) { + SMsgHead *pHead = (SMsgHead *)pCont; + pHead->vgId = htonl(pHead->vgId); + pHead->contLen = htonl(pHead->contLen); + + assert(pHead->contLen > 0); + SVnode *pVnode = vnodeAcquireNotClose(pHead->vgId); + if (pVnode != NULL) { + code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; + vnodeRelease(pVnode); + } + + leftLen -= pHead->contLen; + pCont -= pHead->contLen; + } + + if (queuedMsgNum == 0) { + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code}; + rpcSendResponse(&rpcRsp); + } + + rpcFreeCont(pMsg->pCont); +} + +static void vnodeInitReadMsgFp() { + tsVread.msgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg; + tsVread.msgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg; +} + +static int32_t vnodeProcessReadStart(SVnode *pVnode, SReadMsg *pRead, int32_t qtype) { + int32_t msgType = pRead->msgType; + if (tsVread.msgFp[msgType] == NULL) { + vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]); + return TSDB_CODE_VND_MSG_NOT_PROCESSED; + } else { + vTrace("msg:%p, app:%p type:%s will be processed", pRead, pRead->rpcAhandle, taosMsg[msgType]); + } + + return (*tsVread.msgFp[msgType])(pVnode, pRead); +} + +static void vnodeSendReadRsp(SReadMsg *pRead, int32_t code) { + SRpcMsg rpcRsp = { + .handle = pRead->rpcHandle, + .pCont = pRead->rspRet.rsp, + .contLen = pRead->rspRet.len, + .code = code, + }; + + rpcSendResponse(&rpcRsp); +} + +static void vnodeProcessReadEnd(SVnode *pVnode, SReadMsg *pRead, int32_t qtype, int32_t code) { + if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { + vnodeSendReadRsp(pRead, code); + } else { + if (code == TSDB_CODE_QRY_HAS_RSP) { + vnodeSendReadRsp(pRead, pRead->code); + } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client + assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5)); + } + } + + vnodeFreeFromRQueue(pVnode, pRead); +} + +int32_t vnodeInitRead() { + vnodeInitReadMsgFp(); + + int32_t maxFetchThreads = 4; + float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1); + + SWorkerPool *pPool = &tsVread.query; + pPool->name = "vquery"; + pPool->startFp = (ProcessStartFp)vnodeProcessReadStart; + pPool->endFp = (ProcessEndFp)vnodeProcessReadEnd; + pPool->min = (int32_t)threadsForQuery; + pPool->max = pPool->min; + if (tWorkerInit(pPool) != 0) return -1; + + pPool = &tsVread.fetch; + pPool->name = "vfetch"; + pPool->startFp = (ProcessStartFp)vnodeProcessReadStart; + pPool->endFp = (ProcessEndFp)vnodeProcessReadEnd; + pPool->min = MIN(maxFetchThreads, tsNumOfCores); + pPool->max = pPool->min; + if (tWorkerInit(pPool) != 0) return -1; + + vInfo("vread is initialized, max worker %d", pPool->max); + return 0; +} + +void vnodeCleanupRead() { + tWorkerCleanup(&tsVread.fetch); + tWorkerCleanup(&tsVread.query); + vInfo("vread is closed"); +} + +taos_queue vnodeAllocQueryQueue(SVnode *pVnode) { return tWorkerAllocQueue(&tsVread.query, pVnode); } + +taos_queue vnodeAllocFetchQueue(SVnode *pVnode) { return tWorkerAllocQueue(&tsVread.fetch, pVnode); } + +void vnodeFreeQueryQueue(taos_queue pQueue) { tWorkerFreeQueue(&tsVread.query, pQueue); } + +void vnodeFreeFetchQueue(taos_queue pQueue) { tWorkerFreeQueue(&tsVread.fetch, pQueue); } diff --git a/src/vnode/src/vnodeRead.c b/source/server/vnode/src/vnodeReadMsg.c similarity index 53% rename from src/vnode/src/vnodeRead.c rename to source/server/vnode/src/vnodeReadMsg.c index 64f87ba5caddbb8b9cf90c2a13fa4029a9821ab0..8a0f4b2e0fb229cc100f1fd140310c38cec53dc0 100644 --- a/src/vnode/src/vnodeRead.c +++ b/source/server/vnode/src/vnodeReadMsg.c @@ -16,155 +16,26 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" -#include "tqueue.h" #include "tglobal.h" -#include "query.h" +// #include "query.h" #include "vnodeStatus.h" +#include "vnodeRead.h" +#include "vnodeReadMsg.h" -int32_t vNumOfExistedQHandle; // current initialized and existed query handle in current dnode - -static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead); -static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead); -static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead); - -static int32_t vnodeNotifyCurrentQhandle(void* handle, uint64_t qId, void* qhandle, int32_t vgId); - -int32_t vnodeInitRead(void) { - vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg; - vnodeProcessReadMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg; - return 0; -} - -void vnodeCleanupRead() {} - -// -// After the fetch request enters the vnode queue, if the vnode cannot provide services, the process function are -// still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the -// request enters the queue -// -int32_t vnodeProcessRead(void *vparam, SVReadMsg *pRead) { - SVnodeObj *pVnode = vparam; - int32_t msgType = pRead->msgType; - - if (vnodeProcessReadMsgFp[msgType] == NULL) { - vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]); - return TSDB_CODE_VND_MSG_NOT_PROCESSED; - } - - return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead); -} - -static int32_t vnodeCheckRead(SVnodeObj *pVnode) { - if (!vnodeInReadyStatus(pVnode)) { - vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], - pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; - } - - // tsdb may be in reset state - if (pVnode->tsdb == NULL) { - vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; - } - - if (pVnode->role == TAOS_SYNC_ROLE_MASTER) { - return TSDB_CODE_SUCCESS; - } - - if (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE) { - return TSDB_CODE_SUCCESS; - } - - vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p, cant provide query service", pVnode->vgId, pVnode->syncCfg.replica, - syncRole[pVnode->role], pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; -} - -void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) { - SVnodeObj *pVnode = vparam; - - atomic_sub_fetch_32(&pVnode->queuedRMsg, 1); - vTrace("vgId:%d, free from vrqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedRMsg); - - taosFreeQitem(pRead); - vnodeRelease(pVnode); -} - -static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, int8_t qtype, SRpcMsg *pRpcMsg) { - int32_t size = sizeof(SVReadMsg) + contLen; - SVReadMsg *pRead = taosAllocateQitem(size); - if (pRead == NULL) { - terrno = TSDB_CODE_VND_OUT_OF_MEMORY; - return NULL; - } - - if (pRpcMsg != NULL) { - pRead->rpcHandle = pRpcMsg->handle; - pRead->rpcAhandle = pRpcMsg->ahandle; - pRead->msgType = pRpcMsg->msgType; - pRead->code = pRpcMsg->code; - } - - if (contLen != 0) { - pRead->contLen = contLen; - memcpy(pRead->pCont, pCont, contLen); - } else { - pRead->qhandle = pCont; - } - - pRead->qtype = qtype; - atomic_add_fetch_32(&pVnode->refCount, 1); - - return pRead; -} - -int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { - SVnodeObj *pVnode = vparam; - if (pVnode->dropped) { - return TSDB_CODE_APP_NOT_READY; - } - - SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); - if (pRead == NULL) { - assert(terrno != 0); - return terrno; - } - - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) { - taosFreeQitem(pRead); - vnodeRelease(pVnode); - return code; - } - - atomic_add_fetch_32(&pVnode->queuedRMsg, 1); - - if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRead->msgType == TSDB_MSG_TYPE_FETCH) { - vTrace("vgId:%d, write into vfetch queue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, - pVnode->queuedRMsg); - return taosWriteQitem(pVnode->fqueue, qtype, pRead); - } else { - vTrace("vgId:%d, write into vquery queue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, - pVnode->queuedRMsg); - return taosWriteQitem(pVnode->qqueue, qtype, pRead); - } -} - -static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { - SRpcMsg rpcMsg = {0}; - rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; - rpcMsg.ahandle = ahandle; - - int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg); - if (code == TSDB_CODE_SUCCESS) { - vTrace("QInfo:%p add to vread queue for exec query", *qhandle); - } +#if 0 +// notify connection(handle) that current qhandle is created, if current connection from +// client is broken, the query needs to be killed immediately. +static int32_t vnodeNotifyCurrentQhandle(void *handle, uint64_t qId, void *qhandle, int32_t vgId) { + SRetrieveTableMsg *pMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); + pMsg->qId = htobe64(qId); + pMsg->header.vgId = htonl(vgId); + pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); - return code; + vTrace("QInfo:0x%" PRIx64 "-%p register qhandle to connect:%p", qId, qhandle, handle); + return rpcReportProgress(handle, (char *)pMsg, sizeof(SRetrieveTableMsg)); } /** - * * @param pRet response message object * @param pVnode the vnode object * @param handle qhandle for executing query @@ -172,14 +43,16 @@ static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void * @param ahandle sqlObj address at client side * @return */ -static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, uint64_t qId, void **handle, bool *freeHandle, void *ahandle) { +static int32_t vnodeDumpQueryResult(SVnRsp *pRet, void *pVnode, uint64_t qId, void **handle, bool *freeHandle, + void *ahandle) { bool continueExec = false; int32_t code = TSDB_CODE_SUCCESS; - if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { + if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == + TSDB_CODE_SUCCESS) { if (continueExec) { *freeHandle = false; - code = vnodePutItemIntoReadQueue(pVnode, handle, ahandle); + code = vnodeReputPutToRQueue(pVnode, handle, ahandle); if (code != TSDB_CODE_SUCCESS) { *freeHandle = true; return code; @@ -188,7 +61,7 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, uint64_t qId, v } } else { *freeHandle = true; - vTrace("QInfo:0x%"PRIx64"-%p exec completed, free handle:%d", qId, *handle, *freeHandle); + vTrace("QInfo:0x%" PRIx64 "-%p exec completed, free handle:%d", qId, *handle, *freeHandle); } } else { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -203,7 +76,7 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, uint64_t qId, v return code; } -static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) { +static void vnodeBuildNoResultQueryRsp(SVnRsp *pRet) { pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); pRet->len = sizeof(SRetrieveTableRsp); @@ -212,15 +85,16 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) { pRsp->completed = true; } +#endif - -static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { +int32_t vnodeProcessQueryMsg(SVnode *pVnode, SReadMsg *pRead) { +#if 0 void * pCont = pRead->pCont; int32_t contLen = pRead->contLen; - SRspRet *pRet = &pRead->rspRet; + SVnRsp *pRet = &pRead->rspRet; SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont; - memset(pRet, 0, sizeof(SRspRet)); + memset(pRet, 0, sizeof(SVnRsp)); // qHandle needs to be freed correctly if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { @@ -231,13 +105,13 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { void ** handle = NULL; if (contLen != 0) { - qinfo_t pQInfo = NULL; + qinfo_t pQInfo = NULL; uint64_t qId = genQueryId(); code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, qId); SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; - pRsp->qId = 0; + pRsp->qId = 0; pRet->len = sizeof(SQueryTableRsp); pRet->rsp = pRsp; @@ -250,8 +124,8 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { pRsp->code = terrno; terrno = 0; - vError("vgId:%d, QInfo:0x%"PRIx64 "-%p register qhandle failed, return to app, code:%s,", pVnode->vgId, qId, (void *)pQInfo, - tstrerror(pRsp->code)); + vError("vgId:%d, QInfo:0x%" PRIx64 "-%p register qhandle failed, return to app, code:%s,", pVnode->vgId, qId, + (void *)pQInfo, tstrerror(pRsp->code)); qDestroyQueryInfo(pQInfo); // destroy it directly return pRsp->code; } else { @@ -261,7 +135,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { if (handle != NULL && vnodeNotifyCurrentQhandle(pRead->rpcHandle, qId, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:0x%"PRIx64 "-%p, query discarded since link is broken, %p", pVnode->vgId, qId, *handle, + vError("vgId:%d, QInfo:0x%" PRIx64 "-%p, query discarded since link is broken, %p", pVnode->vgId, qId, *handle, pRead->rpcHandle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; @@ -274,8 +148,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } if (handle != NULL) { - vTrace("vgId:%d, QInfo:0x%"PRIx64 "-%p, dnode query msg disposed, create qhandle and returns to app", vgId, qId, *handle); - code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle); + vTrace("vgId:%d, QInfo:0x%" PRIx64 "-%p, query msg disposed, create qhandle and returns to app", vgId, qId, + *handle); + code = vnodeReputPutToRQueue(pVnode, handle, pRead->rpcHandle); if (code != TSDB_CODE_SUCCESS) { pRsp->code = code; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -283,14 +158,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } } - int32_t remain = atomic_add_fetch_32(&vNumOfExistedQHandle, 1); + int32_t remain = atomic_add_fetch_32(&pVnode->numOfExistQHandle, 1); vTrace("vgId:%d, new qhandle created, total qhandle:%d", pVnode->vgId, remain); } else { assert(pCont != NULL); - void **qhandle = (void **)pRead->qhandle; + void ** qhandle = (void **)pRead->qhandle; uint64_t qId = 0; - vTrace("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); + vTrace("vgId:%d, QInfo:%p, continues to exec query", pVnode->vgId, *qhandle); // In the retrieve blocking model, only 50% CPU will be used in query processing if (tsRetrieveBlockingModel) { @@ -315,10 +190,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; } else { - //void *h1 = qGetResultRetrieveMsg(*qhandle); + // void *h1 = qGetResultRetrieveMsg(*qhandle); - /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */ - //assert(h1 == NULL); + /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then + * FETCH thread execute twice before query thread reach here */ + // assert(h1 == NULL); freehandle = qQueryCompleted(*qhandle); } @@ -327,22 +203,24 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // If the building of result is not required, simply free it. Otherwise, mandatorily free the qhandle if (freehandle || (!buildRes)) { if (freehandle) { - int32_t remain = atomic_sub_fetch_32(&vNumOfExistedQHandle, 1); + int32_t remain = atomic_sub_fetch_32(&pVnode->numOfExistQHandle, 1); vTrace("vgId:%d, QInfo:%p, start to free qhandle, remain qhandle:%d", pVnode->vgId, *qhandle, remain); } qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); } - } } return code; +#endif + return 0; } -static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { - void *pCont = pRead->pCont; - SRspRet *pRet = &pRead->rspRet; +int32_t vnodeProcessFetchMsg(SVnode *pVnode, SReadMsg *pRead) { +#if 0 + void * pCont = pRead->pCont; + SVnRsp *pRet = &pRead->rspRet; SRetrieveTableMsg *pRetrieve = pCont; pRetrieve->free = htons(pRetrieve->free); @@ -351,7 +229,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { vTrace("vgId:%d, qId:0x%" PRIx64 ", retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, pRetrieve->qId, pRetrieve->free, pRead->rpcHandle); - memset(pRet, 0, sizeof(SRspRet)); + memset(pRet, 0, sizeof(SVnRsp)); terrno = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS; @@ -364,16 +242,17 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } if (code != TSDB_CODE_SUCCESS) { - vError("vgId:%d, invalid qId in retrieving result, code:%s, QInfo:%" PRIu64, pVnode->vgId, tstrerror(code), pRetrieve->qId); + vError("vgId:%d, invalid qId in retrieving result, code:%s, QInfo:%" PRIu64, pVnode->vgId, tstrerror(code), + pRetrieve->qId); vnodeBuildNoResultQueryRsp(pRet); return code; } // kill current query and free corresponding resources. if (pRetrieve->free == 1) { - int32_t remain = atomic_sub_fetch_32(&vNumOfExistedQHandle, 1); - vWarn("vgId:%d, QInfo:%"PRIx64 "-%p, retrieve msg received to kill query and free qhandle, remain qhandle:%d", pVnode->vgId, pRetrieve->qId, - *handle, remain); + int32_t remain = atomic_sub_fetch_32(&pVnode->numOfExistQHandle, 1); + vWarn("vgId:%d, QInfo:%" PRIx64 "-%p, retrieve msg received to kill query and free qhandle, remain qhandle:%d", + pVnode->vgId, pRetrieve->qId, *handle, remain); qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -385,9 +264,9 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // register the qhandle to connect to quit query immediate if connection is broken if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, pRetrieve->qId, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - int32_t remain = atomic_sub_fetch_32(&vNumOfExistedQHandle, 1); - vError("vgId:%d, QInfo:%"PRIu64 "-%p, retrieve discarded since link is broken, conn:%p, remain qhandle:%d", pVnode->vgId, pRetrieve->qhandle, - *handle, pRead->rpcHandle, remain); + int32_t remain = atomic_sub_fetch_32(&pVnode->numOfExistQHandle, 1); + vError("vgId:%d, QInfo:%" PRIu64 "-%p, retrieve discarded since link is broken, conn:%p, remain qhandle:%d", + pVnode->vgId, pRetrieve->qhandle, *handle, pRead->rpcHandle, remain); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); @@ -422,29 +301,13 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // If qhandle is not added into vread queue, the query should be completed already or paused with error. // Here free qhandle immediately if (freeHandle) { - int32_t remain = atomic_sub_fetch_32(&vNumOfExistedQHandle, 1); + int32_t remain = atomic_sub_fetch_32(&pVnode->numOfExistQHandle, 1); vTrace("vgId:%d, QInfo:%p, start to free qhandle, remain qhandle:%d", pVnode->vgId, *handle, remain); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); } return code; +#endif + return 0; } -// notify connection(handle) that current qhandle is created, if current connection from -// client is broken, the query needs to be killed immediately. -int32_t vnodeNotifyCurrentQhandle(void *handle, uint64_t qId, void *qhandle, int32_t vgId) { - SRetrieveTableMsg *pMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); - pMsg->qId = htobe64(qId); - pMsg->header.vgId = htonl(vgId); - pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); - - vTrace("QInfo:0x%"PRIx64"-%p register qhandle to connect:%p", qId, qhandle, handle); - return rpcReportProgress(handle, (char *)pMsg, sizeof(SRetrieveTableMsg)); -} - -void vnodeWaitReadCompleted(SVnodeObj *pVnode) { - while (pVnode->queuedRMsg > 0) { - vTrace("vgId:%d, queued rmsg num:%d", pVnode->vgId, pVnode->queuedRMsg); - taosMsleep(10); - } -} diff --git a/src/vnode/src/vnodeStatus.c b/source/server/vnode/src/vnodeStatus.c similarity index 61% rename from src/vnode/src/vnodeStatus.c rename to source/server/vnode/src/vnodeStatus.c index 1eaddc3d25fef5d8753ef1f645413326360fa101..b0b3bbfa498c7435f9c90d38a53bcd6e9a06e5ff 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/source/server/vnode/src/vnodeStatus.c @@ -16,9 +16,9 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" -#include "query.h" -#include "vnodeStatus.h" +// #include "query.h" #include "vnodeRead.h" +#include "vnodeStatus.h" #include "vnodeWrite.h" char* vnodeStatus[] = { @@ -29,30 +29,32 @@ char* vnodeStatus[] = { "reset" }; -bool vnodeSetInitStatus(SVnodeObj* pVnode) { +bool vnodeSetInitStatus(SVnode* pVnode) { pthread_mutex_lock(&pVnode->statusMutex); pVnode->status = TAOS_VN_STATUS_INIT; pthread_mutex_unlock(&pVnode->statusMutex); return true; } -bool vnodeSetReadyStatus(SVnodeObj* pVnode) { +bool vnodeSetReadyStatus(SVnode* pVnode) { bool set = false; pthread_mutex_lock(&pVnode->statusMutex); if (pVnode->status == TAOS_VN_STATUS_INIT || pVnode->status == TAOS_VN_STATUS_READY || - pVnode->status == TAOS_VN_STATUS_UPDATING || pVnode->status == TAOS_VN_STATUS_RESET) { + pVnode->status == TAOS_VN_STATUS_UPDATING) { pVnode->status = TAOS_VN_STATUS_READY; set = true; } +#if 0 qQueryMgmtReOpen(pVnode->qMgmt); +#endif pthread_mutex_unlock(&pVnode->statusMutex); return set; } -static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { +static bool vnodeSetClosingStatusImp(SVnode* pVnode) { bool set = false; pthread_mutex_lock(&pVnode->statusMutex); @@ -65,7 +67,7 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { return set; } -bool vnodeSetClosingStatus(SVnodeObj* pVnode) { +bool vnodeSetClosingStatus(SVnode* pVnode) { if (pVnode->status == TAOS_VN_STATUS_CLOSING) return true; @@ -73,15 +75,17 @@ bool vnodeSetClosingStatus(SVnodeObj* pVnode) { taosMsleep(1); } +#if 0 // release local resources only after cutting off outside connections qQueryMgmtNotifyClosed(pVnode->qMgmt); +#endif vnodeWaitReadCompleted(pVnode); vnodeWaitWriteCompleted(pVnode); return true; } -bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) { +bool vnodeSetUpdatingStatus(SVnode* pVnode) { bool set = false; pthread_mutex_lock(&pVnode->statusMutex); @@ -94,35 +98,7 @@ bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) { return set; } -static bool vnodeSetResetStatusImp(SVnodeObj* pVnode) { - bool set = false; - pthread_mutex_lock(&pVnode->statusMutex); - - if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) { - pVnode->status = TAOS_VN_STATUS_RESET; - set = true; - } - - pthread_mutex_unlock(&pVnode->statusMutex); - return set; -} - -bool vnodeSetResetStatus(SVnodeObj* pVnode) { - while (!vnodeSetResetStatusImp(pVnode)) { - taosMsleep(1); - } - - vInfo("vgId:%d, set to reset status", pVnode->vgId); - - // release local resources only after cutting off outside connections - qQueryMgmtNotifyClosed(pVnode->qMgmt); - vnodeWaitReadCompleted(pVnode); - vnodeWaitWriteCompleted(pVnode); - - return true; -} - -bool vnodeInInitStatus(SVnodeObj* pVnode) { +bool vnodeInInitStatus(SVnode* pVnode) { bool in = false; pthread_mutex_lock(&pVnode->statusMutex); @@ -134,7 +110,7 @@ bool vnodeInInitStatus(SVnodeObj* pVnode) { return in; } -bool vnodeInReadyStatus(SVnodeObj* pVnode) { +bool vnodeInReadyStatus(SVnode* pVnode) { bool in = false; pthread_mutex_lock(&pVnode->statusMutex); @@ -146,19 +122,7 @@ bool vnodeInReadyStatus(SVnodeObj* pVnode) { return in; } -bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode) { - bool in = false; - pthread_mutex_lock(&pVnode->statusMutex); - - if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_UPDATING) { - in = true; - } - - pthread_mutex_unlock(&pVnode->statusMutex); - return in; -} - -bool vnodeInClosingStatus(SVnodeObj* pVnode) { +bool vnodeInClosingStatus(SVnode* pVnode) { bool in = false; pthread_mutex_lock(&pVnode->statusMutex); @@ -170,14 +134,3 @@ bool vnodeInClosingStatus(SVnodeObj* pVnode) { return in; } -bool vnodeInResetStatus(SVnodeObj* pVnode) { - bool in = false; - pthread_mutex_lock(&pVnode->statusMutex); - - if (pVnode->status == TAOS_VN_STATUS_RESET) { - in = true; - } - - pthread_mutex_unlock(&pVnode->statusMutex); - return in; -} diff --git a/src/vnode/src/vnodeVersion.c b/source/server/vnode/src/vnodeVersion.c similarity index 96% rename from src/vnode/src/vnodeVersion.c rename to source/server/vnode/src/vnodeVersion.c index d1aee5a3d39ffbdcbd72aa5459e4bd64858c4bf4..c33053388520b696ee373bd7c22cc3b8dfa885e9 100644 --- a/src/vnode/src/vnodeVersion.c +++ b/source/server/vnode/src/vnodeVersion.c @@ -19,7 +19,7 @@ #include "tglobal.h" #include "vnodeVersion.h" -int32_t vnodeReadVersion(SVnodeObj *pVnode) { +int32_t vnodeReadVersion(SVnode *pVnode) { int32_t len = 0; int32_t maxLen = 100; char * content = calloc(1, maxLen + 1); @@ -71,7 +71,7 @@ PARSE_VER_ERROR: return terrno; } -int32_t vnodeSaveVersion(SVnodeObj *pVnode) { +int32_t vnodeSaveVersion(SVnode *pVnode) { char file[TSDB_FILENAME_LEN + 30] = {0}; sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); @@ -90,7 +90,7 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - taosFsync(fileno(fp)); + taosFsyncFile(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/source/server/vnode/src/vnodeWorker.c b/source/server/vnode/src/vnodeWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..4a8a3a7049e6f88651a5b2580da5db75c5bd895d --- /dev/null +++ b/source/server/vnode/src/vnodeWorker.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "vnodeMain.h" +#include "vnodeWorker.h" + +enum { CLEANUP_TASK = 0, DESTROY_TASK = 1, BACKUP_TASK = 2 }; + +typedef struct { + int32_t vgId; + int32_t code; + int32_t type; + void * rpcHandle; + SVnode *pVnode; +} SVnTask; + +static struct { + SWorkerPool pool; + taos_queue pQueue; +} tsVworker = {0}; + +static void vnodeProcessTaskStart(void *unused, SVnTask *pTask, int32_t qtype) { + pTask->code = 0; + + switch (pTask->type) { + case CLEANUP_TASK: + vnodeCleanUp(pTask->pVnode); + break; + case DESTROY_TASK: + vnodeDestroy(pTask->pVnode); + break; + case BACKUP_TASK: + vnodeBackup(pTask->vgId); + break; + default: + break; + } +} + +static void vnodeProcessTaskEnd(void *unused, SVnTask *pTask, int32_t qtype, int32_t code) { + if (pTask->rpcHandle != NULL) { + SRpcMsg rpcRsp = {.handle = pTask->rpcHandle, .code = pTask->code}; + rpcSendResponse(&rpcRsp); + } + + taosFreeQitem(pTask); +} + +static int32_t vnodeWriteIntoTaskQueue(SVnode *pVnode, int32_t type, void *rpcHandle) { + SVnTask *pTask = taosAllocateQitem(sizeof(SVnTask)); + if (pTask == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY; + + pTask->vgId = pVnode->vgId; + pTask->pVnode = pVnode; + pTask->rpcHandle = rpcHandle; + pTask->type = type; + + + return taosWriteQitem(tsVworker.pQueue, TAOS_QTYPE_RPC, pTask); +} + +void vnodeProcessCleanupTask(SVnode *pVnode) { + vnodeWriteIntoTaskQueue(pVnode, CLEANUP_TASK, NULL); +} + +void vnodeProcessDestroyTask(SVnode *pVnode) { + vnodeWriteIntoTaskQueue(pVnode, DESTROY_TASK, NULL); +} + +void vnodeProcessBackupTask(SVnode *pVnode) { + vnodeWriteIntoTaskQueue(pVnode, BACKUP_TASK, NULL); +} + +int32_t vnodeInitWorker() { + SWorkerPool *pPool = &tsVworker.pool; + pPool->name = "vworker"; + pPool->startFp = (ProcessStartFp)vnodeProcessTaskStart; + pPool->endFp = (ProcessEndFp)vnodeProcessTaskEnd; + pPool->min = 0; + pPool->max = 1; + if (tWorkerInit(pPool) != 0) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + tsVworker.pQueue = tWorkerAllocQueue(pPool, NULL); + + vInfo("vworker is initialized, max worker %d", pPool->max); + return TSDB_CODE_SUCCESS; +} + +void vnodeCleanupWorker() { + tWorkerFreeQueue(&tsVworker.pool, tsVworker.pQueue); + tWorkerCleanup(&tsVworker.pool); + tsVworker.pQueue = NULL; + vInfo("vworker is closed"); +} diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c index d4ef4ab3370d8f6196e995407662235b68efeafa..3c2634a2cfe76a79bdef4a142abdad2400120adb 100644 --- a/source/server/vnode/src/vnodeWrite.c +++ b/source/server/vnode/src/vnodeWrite.c @@ -13,55 +13,224 @@ * along with this program. If not, see . */ -#include "vnodeInt.h" +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "tqueue.h" +#include "tworker.h" +#include "taosmsg.h" +#include "vnodeMain.h" +#include "vnodeStatus.h" +#include "vnodeWrite.h" +#include "vnodeWriteMsg.h" -int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { - // TODO: Check inputs +typedef int32_t (*WriteMsgFp)(SVnode *, void *pCont, SVnRsp *); -#if 0 - void *pMem = NULL; - if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { - // No more memory to allocate, schedule an async commit - // and continue - vnodeAsyncCommit(pVnode); - - // Reset allocator and allocat more - vnodeResetAllocator(pVnode); - pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq)); - if (pMem == NULL) { - // TODO: handle the error - } +typedef struct { + int32_t code; + int8_t qtype; + SVnode * pVnode; + SRpcMsg rpcMsg; + SVnRsp rspRet; + char reserveForSync[24]; + SWalHead walHead; +} SVnWriteMsg; + +static struct { + SWriteWorkerPool pool; + int64_t queuedBytes; + int32_t queuedMsgs; +} tsVwrite = {0}; + +void vnodeStartWrite(SVnode *pVnode) {} +void vnodeStoprite(SVnode *pVnode) {} + +void vnodeWaitWriteCompleted(SVnode *pVnode) { + while (pVnode->queuedWMsg > 0) { + vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); + taosMsleep(10); + } +} + +static int32_t vnodeWriteToWQueue(SVnode *pVnode, SWalHead *pHead, int32_t qtype, SRpcMsg *pRpcMsg) { + if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { + vWarn("vgId:%d, no write auth", pVnode->vgId); + return TSDB_CODE_VND_NO_WRITE_AUTH; } - // TODO: if SSubmitReq is compressed or encoded, we need to decode the request - memcpy(pMem, pReq, REQ_SIZE(pReq)); + if (tsAvailDataDirGB <= tsMinimalDataDirGB) { + vWarn("vgId:%d, failed to write into vwqueue since no diskspace, avail:%fGB", pVnode->vgId, tsAvailDataDirGB); + return TSDB_CODE_VND_NO_DISKSPACE; + } - if (tqPushMsg((SSubmitReq *)pReq) < 0) { - // TODO: handle error + if (pHead->len > TSDB_MAX_WAL_SIZE) { + vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version); + return TSDB_CODE_WAL_SIZE_LIMIT; } - SSubmitReqReader reader; - taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem); + if (!vnodeInReadyStatus(pVnode)) { + vError("vgId:%d, failed to write into vwqueue, vstatus is %s", pVnode->vgId, vnodeStatus[pVnode->status]); + return TSDB_CODE_APP_NOT_READY; + } - if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) { - // TODO: handler error + if (tsVwrite.queuedBytes > tsMaxVnodeQueuedBytes) { + vDebug("vgId:%d, too many bytes:%" PRId64 " in vwqueue, flow control", pVnode->vgId, tsVwrite.queuedBytes); + return TSDB_CODE_VND_IS_FLOWCTRL; } -#endif - return 0; + int32_t size = sizeof(SVnWriteMsg) + pHead->len; + SVnWriteMsg *pWrite = taosAllocateQitem(size); + if (pWrite == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + if (pRpcMsg != NULL) { + pWrite->rpcMsg = *pRpcMsg; + } + + memcpy(&pWrite->walHead, pHead, sizeof(SWalHead) + pHead->len); + pWrite->pVnode = pVnode; + pWrite->qtype = qtype; + + atomic_add_fetch_64(&tsVwrite.queuedBytes, size); + atomic_add_fetch_32(&tsVwrite.queuedMsgs, 1); + atomic_add_fetch_32(&pVnode->refCount, 1); + atomic_add_fetch_32(&pVnode->queuedWMsg, 1); + taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); + + return TSDB_CODE_SUCCESS; +} + +static void vnodeFreeFromWQueue(SVnode *pVnode, SVnWriteMsg *pWrite) { + int64_t size = sizeof(SVnWriteMsg) + pWrite->walHead.len; + atomic_sub_fetch_64(&tsVwrite.queuedBytes, size); + atomic_sub_fetch_32(&tsVwrite.queuedMsgs, 1); + atomic_sub_fetch_32(&pVnode->queuedWMsg, 1); + + taosFreeQitem(pWrite); + vnodeRelease(pVnode); +} + +int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead) { + return vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_WAL, NULL); +} + +void vnodeProcessWriteMsg(SRpcMsg *pRpcMsg) { + int32_t code; + + SMsgHead *pMsg = pRpcMsg->pCont; + pMsg->vgId = htonl(pMsg->vgId); + pMsg->contLen = htonl(pMsg->contLen); + + SVnode *pVnode = vnodeAcquireNotClose(pMsg->vgId); + if (pVnode == NULL) { + code = TSDB_CODE_VND_INVALID_VGROUP_ID; + } else { + SWalHead *pHead = (SWalHead *)((char *)pRpcMsg->pCont - sizeof(SWalHead)); + pHead->msgType = pRpcMsg->msgType; + pHead->version = 0; + pHead->len = pMsg->contLen; + code = vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg); + } + + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rpcRsp = {.handle = pRpcMsg->handle, .code = code}; + rpcSendResponse(&rpcRsp); + } + + vnodeRelease(pVnode); + rpcFreeCont(pRpcMsg->pCont); +} + +static bool vnodeProcessWriteStart(SVnode *pVnode, SVnWriteMsg *pWrite, int32_t qtype) { + SWalHead *pHead = &pWrite->walHead; + SVnRsp * pRet = &pWrite->rspRet; + int32_t msgType = pHead->msgType; + + vTrace("vgId:%d, msg:%s will be processed, hver:%" PRIu64, pVnode->vgId, taosMsg[pHead->msgType], pHead->version); + + // write into WAL +#if 0 + pWrite->code = walWrite(pVnode->wal, pHead); + if (pWrite->code < 0) return false; +#endif + + pVnode->version = pHead->version; + + // write data locally + switch (msgType) { + case TSDB_MSG_TYPE_SUBMIT: + pRet->len = sizeof(SSubmitRsp); + pRet->rsp = rpcMallocCont(pRet->len); + pWrite->code = vnodeProcessSubmitReq(pVnode, (void*)pHead->cont, pRet->rsp); + break; + case TSDB_MSG_TYPE_MD_CREATE_TABLE: + pWrite->code = vnodeProcessCreateTableReq(pVnode, (void*)pHead->cont, NULL); + break; + case TSDB_MSG_TYPE_MD_DROP_TABLE: + pWrite->code = vnodeProcessDropTableReq(pVnode, (void*)pHead->cont, NULL); + break; + case TSDB_MSG_TYPE_MD_ALTER_TABLE: + pWrite->code = vnodeProcessAlterTableReq(pVnode, (void*)pHead->cont, NULL); + break; + case TSDB_MSG_TYPE_MD_DROP_STABLE: + pWrite->code = vnodeProcessDropStableReq(pVnode, (void*)pHead->cont, NULL); + break; + case TSDB_MSG_TYPE_UPDATE_TAG_VAL: + pWrite->code = vnodeProcessUpdateTagValReq(pVnode, (void*)pHead->cont, NULL); + break; + default: + pWrite->code = TSDB_CODE_VND_MSG_NOT_PROCESSED; + break; + } + + if (pWrite->code < 0) return false; + + // update fync + return (pWrite->code == 0 && msgType != TSDB_MSG_TYPE_SUBMIT); +} + +static void vnodeFsync(SVnode *pVnode, bool fsync) { +#if 0 + walFsync(pVnode->wal, fsync); +#endif } -int vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp) { - // TODO - return 0; +static void vnodeProcessWriteEnd(SVnode *pVnode, SVnWriteMsg *pWrite, int32_t qtype, int32_t code) { + if (qtype == TAOS_QTYPE_RPC) { + SRpcMsg rpcRsp = { + .handle = pWrite->rpcMsg.handle, + .pCont = pWrite->rspRet.rsp, + .contLen = pWrite->rspRet.len, + .code = pWrite->code, + }; + rpcSendResponse(&rpcRsp); + } else { + if (pWrite->rspRet.rsp) { + rpcFreeCont(pWrite->rspRet.rsp); + } + } + vnodeFreeFromWQueue(pVnode, pWrite); } -int vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { - // TODO - return 0; +int32_t vnodeInitWrite() { + SWriteWorkerPool *pPool = &tsVwrite.pool; + pPool->name = "vwrite"; + pPool->max = tsNumOfCores; + pPool->startFp = (ProcessWriteStartFp)vnodeProcessWriteStart; + pPool->syncFp = (ProcessWriteSyncFp)vnodeFsync; + pPool->endFp = (ProcessWriteEndFp)vnodeProcessWriteEnd; + if (tWriteWorkerInit(pPool) != 0) return -1; + + vInfo("vwrite is initialized, max worker %d", pPool->max); + return TSDB_CODE_SUCCESS; } -int vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp) { - // TODO - return 0; +void vnodeCleanupWrite() { + tWriteWorkerCleanup(&tsVwrite.pool); + vInfo("vwrite is closed"); } + +taos_queue vnodeAllocWriteQueue(SVnode *pVnode) { return tWriteWorkerAllocQueue(&tsVwrite.pool, pVnode); } + +void vnodeFreeWriteQueue(taos_queue pQueue) { tWriteWorkerFreeQueue(&tsVwrite.pool, pQueue); } \ No newline at end of file diff --git a/source/server/vnode/src/vnodeWriteMsg.c b/source/server/vnode/src/vnodeWriteMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..0fe6fa2bc948aefe10c1b503b19c1253590a0238 --- /dev/null +++ b/source/server/vnode/src/vnodeWriteMsg.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "vnodeWriteMsg.h" + +int32_t vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { + // TODO: Check inputs + +#if 0 + void *pMem = NULL; + if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { + // No more memory to allocate, schedule an async commit + // and continue + vnodeAsyncCommit(pVnode); + + // Reset allocator and allocat more + vnodeResetAllocator(pVnode); + pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq)); + if (pMem == NULL) { + // TODO: handle the error + } + } + + // TODO: if SSubmitReq is compressed or encoded, we need to decode the request + memcpy(pMem, pReq, REQ_SIZE(pReq)); + + if (tqPushMsg((SSubmitReq *)pReq) < 0) { + // TODO: handle error + } + + SSubmitReqReader reader; + taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem); + + if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) { + // TODO: handler error + } +#endif + + return 0; +} + +int32_t vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp) { + // TODO + return 0; +} + + int32_t vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { + // TODO + return 0; +} + +int32_t vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp) { + // TODO + return 0; +} + +int32_t vnodeProcessDropStableReq(SVnode *pVnode, SDropStableReq *pReq, SDropStableRsp *pRsp) { + // TODO + return 0; +} + +int32_t vnodeProcessUpdateTagValReq(SVnode *pVnode, SUpdateTagValReq *pReq, SUpdateTagValRsp *pRsp) { + // TODO + return 0; +} diff --git a/source/util/src/tstep.c b/source/util/src/tstep.c index b04135194a235723ab5a1774e492afd01d55645a..656ac658a01455f6538eab9f0ac39c9255919738 100644 --- a/source/util/src/tstep.c +++ b/source/util/src/tstep.c @@ -20,15 +20,14 @@ typedef struct SStepObj { char * name; - void ** self; InitFp initFp; CleanupFp cleanupFp; } SStep; typedef struct SSteps { - int32_t cursize; - int32_t maxsize; - SStep * steps; + int32_t cursize; + int32_t maxsize; + SStep * steps; ReportFp reportFp; } SSteps; @@ -44,14 +43,14 @@ SSteps *taosStepInit(int32_t maxsize, ReportFp fp) { return steps; } -int32_t taosStepAdd(struct SSteps *steps, char *name, void **obj, InitFp initFp, CleanupFp cleanupFp) { +int32_t taosStepAdd(struct SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp) { if (steps == NULL) return -1; if (steps->cursize >= steps->maxsize) { uError("failed to add step since up to the maxsize"); return -1; } - SStep step = {.name = name, .self = obj, .initFp = initFp, .cleanupFp = cleanupFp}; + SStep step = {.name = name, .initFp = initFp, .cleanupFp = cleanupFp}; steps->steps[steps->cursize++] = step; return 0; } @@ -61,7 +60,7 @@ static void taosStepCleanupImp(SSteps *steps, int32_t pos) { SStep *step = steps->steps + s; uDebug("step:%s will cleanup", step->name); if (step->cleanupFp != NULL) { - (*step->cleanupFp)(step->self); + (*step->cleanupFp)(); } } } @@ -77,7 +76,7 @@ int32_t taosStepExec(SSteps *steps) { (*steps->reportFp)(step->name, "start initialize"); } - int32_t code = (*step->initFp)(step->self); + int32_t code = (*step->initFp)(); if (code != 0) { uDebug("step:%s will cleanup", step->name); taosStepCleanupImp(steps, s); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 5370e78c090b28580a28e04dbb92b67ffaed3cb1..9e21583895bba8785cea19f417c50e60a50e359b 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -19,37 +19,16 @@ #include "tqueue.h" #include "tworker.h" -static void *taosWorkerThreadFp(void *wparam) { - SWorker * worker = wparam; - SWorkerPool *pool = worker->pool; - void * msg = NULL; - int32_t qtype = 0; - void * ahandle = NULL; - int32_t code = 0; - - setThreadName(pool->name); - - while (1) { - if (taosReadQitemFromQset(pool->qset, &qtype, (void **)&msg, &ahandle) == 0) { - uDebug("pool:%s, worker:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset); - break; - } - - code = (*pool->reqFp)(ahandle, msg); - (*pool->rspFp)(ahandle, msg, qtype, code); - } - - return NULL; -} +typedef void* (*ThreadFp)(void *param); int32_t tWorkerInit(SWorkerPool *pool) { pool->qset = taosOpenQset(); pool->workers = calloc(sizeof(SWorker), pool->max); pthread_mutex_init(&pool->mutex, NULL); for (int i = 0; i < pool->max; ++i) { - SWorker *pWorker = pool->workers + i; - pWorker->id = i; - pWorker->pool = pool; + SWorker *worker = pool->workers + i; + worker->id = i; + worker->pool = pool; } uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); @@ -58,16 +37,16 @@ int32_t tWorkerInit(SWorkerPool *pool) { void tWorkerCleanup(SWorkerPool *pool) { for (int i = 0; i < pool->max; ++i) { - SWorker *pWorker = pool->workers + i; - if(taosCheckPthreadValid(pWorker->thread)) { + SWorker *worker = pool->workers + i; + if (taosCheckPthreadValid(worker->thread)) { taosQsetThreadResume(pool->qset); } } for (int i = 0; i < pool->max; ++i) { - SWorker *pWorker = pool->workers + i; - if (taosCheckPthreadValid(pWorker->thread)) { - pthread_join(pWorker->thread, NULL); + SWorker *worker = pool->workers + i; + if (taosCheckPthreadValid(worker->thread)) { + pthread_join(worker->thread, NULL); } } @@ -78,42 +57,204 @@ void tWorkerCleanup(SWorkerPool *pool) { uInfo("worker:%s is closed", pool->name); } -void *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle) { +static void *tWorkerThreadFp(SWorker *worker) { + SWorkerPool *pool = worker->pool; + + void * msg = NULL; + void * ahandle = NULL; + int32_t qtype = 0; + int32_t code = 0; + + taosBlockSIGPIPE(); + setThreadName(pool->name); + uDebug("worker:%s:%d is running", pool->name, worker->id); + + while (1) { + if (taosReadQitemFromQset(pool->qset, &qtype, (void **)&msg, &ahandle) == 0) { + uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset); + break; + } + + code = (*pool->startFp)(ahandle, msg, qtype); + (*pool->endFp)(ahandle, msg, qtype, code); + } + + return NULL; +} + +taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle) { pthread_mutex_lock(&pool->mutex); - taos_queue pQueue = taosOpenQueue(); - if (pQueue == NULL) { + taos_queue queue = taosOpenQueue(); + if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); return NULL; } - taosAddIntoQset(pool->qset, pQueue, ahandle); + taosAddIntoQset(pool->qset, queue, ahandle); // spawn a thread to process queue if (pool->num < pool->max) { do { - SWorker *pWorker = pool->workers + pool->num; + SWorker *worker = pool->workers + pool->num; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, taosWorkerThreadFp, pWorker) != 0) { - uError("workers:%s:%d failed to create thread to process since %s", pool->name, pWorker->id, strerror(errno)); + if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWorkerThreadFp, worker) != 0) { + uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); } pthread_attr_destroy(&thAttr); pool->num++; - uDebug("workers:%s:%d is launched, total:%d", pool->name, pWorker->id, pool->num); + uDebug("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num); } while (pool->num < pool->min); } pthread_mutex_unlock(&pool->mutex); - uDebug("workers:%s, queue:%p is allocated, ahandle:%p", pool->name, pQueue, ahandle); + uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); + + return queue; +} + +void tWorkerFreeQueue(SWorkerPool *pool, void *queue) { + taosCloseQueue(queue); + uDebug("worker:%s, queue:%p is freed", pool->name, queue); +} + +int32_t tWriteWorkerInit(SWriteWorkerPool *pool) { + pool->nextId = 0; + pool->workers = calloc(sizeof(SWriteWorker), pool->max); + if (pool->workers == NULL) return -1; + + pthread_mutex_init(&pool->mutex, NULL); + for (int32_t i = 0; i < pool->max; ++i) { + SWriteWorker *worker = pool->workers + i; + worker->id = i; + worker->qall = NULL; + worker->qset = NULL; + worker->pool = pool; + } + + uInfo("worker:%s is initialized, max:%d", pool->name, pool->max); + return 0; +} + +void tWriteWorkerCleanup(SWriteWorkerPool *pool) { + for (int32_t i = 0; i < pool->max; ++i) { + SWriteWorker *worker = pool->workers + i; + if (taosCheckPthreadValid(worker->thread)) { + if (worker->qset) taosQsetThreadResume(worker->qset); + } + } + + for (int32_t i = 0; i < pool->max; ++i) { + SWriteWorker *worker = pool->workers + i; + if (taosCheckPthreadValid(worker->thread)) { + pthread_join(worker->thread, NULL); + taosFreeQall(worker->qall); + taosCloseQset(worker->qset); + } + } + + free(pool->workers); + pthread_mutex_destroy(&pool->mutex); + + uInfo("worker:%s is closed", pool->name); +} + +static void *tWriteWorkerThreadFp(SWriteWorker *worker) { + SWriteWorkerPool *pool = worker->pool; + + void * msg = NULL; + void * ahandle = NULL; + int32_t numOfMsgs = 0; + int32_t qtype = 0; + + taosBlockSIGPIPE(); + setThreadName(pool->name); + uDebug("worker:%s:%d is running", pool->name, worker->id); + + while (1) { + numOfMsgs = taosReadAllQitemsFromQset(worker->qset, worker->qall, &ahandle); + if (numOfMsgs == 0) { + uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, worker->qset); + break; + } + + bool fsync = false; + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(worker->qall, &qtype, (void **)&msg); + fsync = fsync | (*pool->startFp)(ahandle, msg, qtype); + } + + (*pool->syncFp)(ahandle, fsync); + + // browse all items, and process them one by one + taosResetQitems(worker->qall); + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(worker->qall, &qtype, (void **)&msg); + (*pool->endFp)(ahandle, msg, qtype); + } + } + + return NULL; +} + +taos_queue tWriteWorkerAllocQueue(SWriteWorkerPool *pool, void *ahandle) { + pthread_mutex_lock(&pool->mutex); + SWriteWorker *worker = pool->workers + pool->nextId; + + taos_queue *queue = taosOpenQueue(); + if (queue == NULL) { + pthread_mutex_unlock(&pool->mutex); + return NULL; + } + + if (worker->qset == NULL) { + worker->qset = taosOpenQset(); + if (worker->qset == NULL) { + taosCloseQueue(queue); + pthread_mutex_unlock(&pool->mutex); + return NULL; + } + + taosAddIntoQset(worker->qset, queue, ahandle); + worker->qall = taosAllocateQall(); + if (worker->qall == NULL) { + taosCloseQset(worker->qset); + taosCloseQueue(queue); + pthread_mutex_unlock(&pool->mutex); + return NULL; + } + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWriteWorkerThreadFp, worker) != 0) { + uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); + taosFreeQall(worker->qall); + taosCloseQset(worker->qset); + taosCloseQueue(queue); + queue = NULL; + } else { + uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); + pool->nextId = (pool->nextId + 1) % pool->max; + } + + pthread_attr_destroy(&thAttr); + } else { + taosAddIntoQset(worker->qset, queue, ahandle); + pool->nextId = (pool->nextId + 1) % pool->max; + } + + pthread_mutex_unlock(&pool->mutex); + uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); - return pQueue; + return queue; } -void tWorkerFreeQueue(SWorkerPool *pool, void *pQueue) { - taosCloseQueue(pQueue); - uDebug("workers:%s, queue:%p is freed", pool->name, pQueue); +void tWriteWorkerFreeQueue(SWriteWorkerPool *pool, taos_queue queue) { + taosCloseQueue(queue); + uDebug("worker:%s, queue:%p is freed", pool->name, queue); } diff --git a/src/inc/twal.h b/src/inc/twal.h deleted file mode 100644 index 868a1fbd780232303b42e58185ffc00730c17546..0000000000000000000000000000000000000000 --- a/src/inc/twal.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_WAL_H_ -#define _TD_WAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum { - TAOS_WAL_NOLOG = 0, - TAOS_WAL_WRITE = 1, - TAOS_WAL_FSYNC = 2 -} EWalType; - -typedef enum { - TAOS_WAL_NOT_KEEP = 0, - TAOS_WAL_KEEP = 1 -} EWalKeep; - -typedef struct { - int8_t msgType; - int8_t sver; // sver 2 for WAL SDataRow/SMemRow compatibility - int8_t reserved[2]; - int32_t len; - uint64_t version; - uint32_t signature; - uint32_t cksum; - char cont[]; -} SWalHead; - -typedef struct { - int32_t vgId; - int32_t fsyncPeriod; // millisecond - EWalType walLevel; // wal level - EWalKeep keep; // keep the wal file when closed -} SWalCfg; - -typedef void * twalh; // WAL HANDLE -typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg); - -int32_t walInit(); -void walCleanUp(); -twalh walOpen(char *path, SWalCfg *pCfg); -int32_t walAlter(twalh pWal, SWalCfg *pCfg); -void walStop(twalh); -void walClose(twalh); -int32_t walRenew(twalh); -void walRemoveOneOldFile(twalh); -void walRemoveAllOldFiles(twalh); -int32_t walWrite(twalh, SWalHead *); -void walFsync(twalh, bool forceFsync); -int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); -int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); -uint64_t walGetVersion(twalh); -void walResetVersion(twalh, uint64_t newVer); - -#ifdef __cplusplus -} -#endif - -#endif // _TD_WAL_H_ diff --git a/src/raft/CMakeLists.txt b/src/raft/CMakeLists.txt deleted file mode 100644 index 3dc66010383ed97ec4c0c5b28170158e5b031a1d..0000000000000000000000000000000000000000 --- a/src/raft/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -aux_source_directory(source RAFT_SRC) -add_library(raft ${RAFT_SRC}) -target_include_directories( - raft - PUBLIC "${CMAKE_SOURCE_DIR}/include/raft" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" -) \ No newline at end of file diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt deleted file mode 100644 index 6238f43d32ad2ed973f522aca3bb5dfca9101435..0000000000000000000000000000000000000000 --- a/src/vnode/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) -INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) - -ADD_LIBRARY(vnode ${SRC}) -TARGET_LINK_LIBRARIES(vnode tsdb tcq common) diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h deleted file mode 100644 index 4864b79dc4ee7c718ad7c023277793f21e74446d..0000000000000000000000000000000000000000 --- a/src/vnode/inc/vnodeInt.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_VNODE_INT_H -#define TDENGINE_VNODE_INT_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "tlog.h" -#include "tsync.h" -#include "tcq.h" -#include "tsdb.h" -#include "vnode.h" - -extern int32_t vDebugFlag; -extern int32_t vNumOfExistedQHandle; // current initialized and existed query handle in current dnode - -#define vFatal(...) { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }} -#define vError(...) { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }} -#define vWarn(...) { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }} -#define vInfo(...) { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("VND ", 255, __VA_ARGS__); }} -#define vDebug(...) { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }} -#define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }} - -typedef struct { - int32_t vgId; // global vnode group ID - int32_t refCount; // reference count - int64_t queuedWMsgSize; - int32_t queuedWMsg; - int32_t queuedRMsg; - int32_t flowctrlLevel; - int8_t preClose; // drop and close switch - int8_t reserved[3]; - int64_t sequence; // for topic - int8_t status; - int8_t role; - int8_t accessState; - int8_t isFull; - int8_t isCommiting; - int8_t dbReplica; - int8_t dropped; - int8_t dbType; - uint64_t version; // current version - uint64_t cversion; // version while commit start - uint64_t fversion; // version on saved data file - void * wqueue; // write queue - void * qqueue; // read query queue - void * fqueue; // read fetch/cancel queue - void * wal; - void * tsdb; - int64_t sync; - void * events; - void * cq; // continuous query - int32_t dbCfgVersion; - int32_t vgCfgVersion; - STsdbCfg tsdbCfg; - SSyncCfg syncCfg; - SWalCfg walCfg; - void * qMgmt; - char * rootDir; - tsem_t sem; - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; - pthread_mutex_t statusMutex; -} SVnodeObj; - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/vnode/inc/vnodeSync.h b/src/vnode/inc/vnodeSync.h deleted file mode 100644 index 28fb63dd6a2db971b430b526b4304a37c3ece9a1..0000000000000000000000000000000000000000 --- a/src/vnode/inc/vnodeSync.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_VNODE_SYNC_H -#define TDENGINE_VNODE_SYNC_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "vnodeInt.h" - -uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver); -int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId); -void vnodeNotifyRole(int32_t vgId, int8_t role); -void vnodeCtrlFlow(int32_t vgId, int32_t level); -void vnodeStartSyncFile(int32_t vgId); -void vnodeStopSyncFile(int32_t vgId, uint64_t fversion); -void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); -int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); -int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver); - -void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/vnode/src/vnodeBackup.c b/src/vnode/src/vnodeBackup.c deleted file mode 100644 index 801af42e0e6869944ec60169b0662131be787cba..0000000000000000000000000000000000000000 --- a/src/vnode/src/vnodeBackup.c +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tutil.h" -#include "tqueue.h" -#include "tglobal.h" -#include "tfs.h" -#include "vnodeBackup.h" -#include "vnodeMain.h" - -typedef struct { - int32_t vgId; -} SVBackupMsg; - -typedef struct { - pthread_t thread; - int32_t workerId; -} SVBackupWorker; - -typedef struct { - int32_t num; - SVBackupWorker *worker; -} SVBackupWorkerPool; - -static SVBackupWorkerPool tsVBackupPool; -static taos_qset tsVBackupQset; -static taos_queue tsVBackupQueue; - -static void vnodeProcessBackupMsg(SVBackupMsg *pMsg) { - int32_t vgId = pMsg->vgId; - char newDir[TSDB_FILENAME_LEN] = {0}; - char stagingDir[TSDB_FILENAME_LEN] = {0}; - - sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); - sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); - - if (tsEnableVnodeBak) { - tfsRmdir(newDir); - tfsRename(stagingDir, newDir); - } else { - vInfo("vgId:%d, vnode backup not enabled", vgId); - - tfsRmdir(stagingDir); - } -} - -static void *vnodeBackupFunc(void *param) { - setThreadName("vnodeBackup"); - - while (1) { - SVBackupMsg *pMsg = NULL; - if (taosReadQitemFromQset(tsVBackupQset, NULL, (void **)&pMsg, NULL) == 0) { - vDebug("qset:%p, vbackup got no message from qset, exiting", tsVBackupQset); - break; - } - - vTrace("vgId:%d, will be processed in vbackup queue", pMsg->vgId); - vnodeProcessBackupMsg(pMsg); - - vTrace("vgId:%d, disposed in vbackup worker", pMsg->vgId); - taosFreeQitem(pMsg); - } - - return NULL; -} - -static int32_t vnodeStartBackup() { - tsVBackupQueue = taosOpenQueue(); - if (tsVBackupQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; - - taosAddIntoQset(tsVBackupQset, tsVBackupQueue, NULL); - - for (int32_t i = 0; i < tsVBackupPool.num; ++i) { - SVBackupWorker *pWorker = tsVBackupPool.worker + i; - pWorker->workerId = i; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&pWorker->thread, &thAttr, vnodeBackupFunc, pWorker) != 0) { - vError("failed to create thread to process vbackup queue, reason:%s", strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - - vDebug("vbackup:%d is launched, total:%d", pWorker->workerId, tsVBackupPool.num); - } - - vDebug("vbackup queue:%p is allocated", tsVBackupQueue); - - return TSDB_CODE_SUCCESS; -} - -static int32_t vnodeWriteIntoBackupWorker(int32_t vgId) { - SVBackupMsg *pMsg = taosAllocateQitem(sizeof(SVBackupMsg)); - if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY; - - pMsg->vgId = vgId; - - int32_t code = taosWriteQitem(tsVBackupQueue, TAOS_QTYPE_RPC, pMsg); - if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS; - - return code; -} - -int32_t vnodeBackup(int32_t vgId) { - vTrace("vgId:%d, will backup", vgId); - return vnodeWriteIntoBackupWorker(vgId); -} - -int32_t vnodeInitBackup() { - tsVBackupQset = taosOpenQset(); - - tsVBackupPool.num = 1; - tsVBackupPool.worker = calloc(sizeof(SVBackupWorker), tsVBackupPool.num); - - if (tsVBackupPool.worker == NULL) return -1; - for (int32_t i = 0; i < tsVBackupPool.num; ++i) { - SVBackupWorker *pWorker = tsVBackupPool.worker + i; - pWorker->workerId = i; - vDebug("vbackup:%d is created", i); - } - - vDebug("vbackup is initialized, num:%d qset:%p", tsVBackupPool.num, tsVBackupQset); - - return vnodeStartBackup(); -} - -void vnodeCleanupBackup() { - for (int32_t i = 0; i < tsVBackupPool.num; ++i) { - SVBackupWorker *pWorker = tsVBackupPool.worker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - taosQsetThreadResume(tsVBackupQset); - } - vDebug("vbackup:%d is closed", i); - } - - for (int32_t i = 0; i < tsVBackupPool.num; ++i) { - SVBackupWorker *pWorker = tsVBackupPool.worker + i; - vDebug("vbackup:%d start to join", i); - if (taosCheckPthreadValid(pWorker->thread)) { - pthread_join(pWorker->thread, NULL); - } - vDebug("vbackup:%d join success", i); - } - - vDebug("vbackup is closed, qset:%p", tsVBackupQset); - - taosCloseQset(tsVBackupQset); - tsVBackupQset = NULL; - - tfree(tsVBackupPool.worker); - - vDebug("vbackup queue:%p is freed", tsVBackupQueue); - taosCloseQueue(tsVBackupQueue); - tsVBackupQueue = NULL; -} diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c deleted file mode 100644 index 8d699cb100675a0c6cfe7b02e5235ab8bb2c82d1..0000000000000000000000000000000000000000 --- a/src/vnode/src/vnodeMgmt.c +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "dnode.h" -#include "vnodeStatus.h" -#include "vnodeBackup.h" -#include "vnodeWorker.h" -#include "vnodeRead.h" -#include "vnodeWrite.h" -#include "vnodeMain.h" - -static SHashObj *tsVnodesHash = NULL; - -static int32_t vnodeInitHash(void); -static void vnodeCleanupHash(void); -static void vnodeIncRef(void *ptNode); - -static SStep tsVnodeSteps[] = { - {"vnode-backup", vnodeInitBackup, vnodeCleanupBackup}, - {"vnode-worker", vnodeInitMWorker, vnodeCleanupMWorker}, - {"vnode-write", vnodeInitWrite, vnodeCleanupWrite}, - {"vnode-read", vnodeInitRead, vnodeCleanupRead}, - {"vnode-hash", vnodeInitHash, vnodeCleanupHash}, - {"tsdb-queue", tsdbInitCommitQueue, tsdbDestroyCommitQueue} -}; - -int32_t vnodeInitMgmt() { - int32_t stepSize = sizeof(tsVnodeSteps) / sizeof(SStep); - return dnodeStepInit(tsVnodeSteps, stepSize); -} - -void vnodeCleanupMgmt() { - int32_t stepSize = sizeof(tsVnodeSteps) / sizeof(SStep); - dnodeStepCleanup(tsVnodeSteps, stepSize); -} - -static int32_t vnodeInitHash() { - tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (tsVnodesHash == NULL) { - vError("failed to init vnode mgmt"); - return -1; - } - - return 0; -} - -static void vnodeCleanupHash() { - if (tsVnodesHash != NULL) { - vDebug("vnode mgmt is cleanup"); - taosHashCleanup(tsVnodesHash); - tsVnodesHash = NULL; - } -} - -void *vnodeGetWal(void *pVnode) { - return ((SVnodeObj *)pVnode)->wal; -} - -void vnodeAddIntoHash(SVnodeObj *pVnode) { - taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); -} - -void vnodeRemoveFromHash(SVnodeObj *pVnode) { - taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t)); -} - -static void vnodeIncRef(void *ptNode) { - assert(ptNode != NULL); - - SVnodeObj **ppVnode = (SVnodeObj **)ptNode; - assert(ppVnode); - assert(*ppVnode); - - SVnodeObj *pVnode = *ppVnode; - atomic_add_fetch_32(&pVnode->refCount, 1); - vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); -} - -void *vnodeAcquire(int32_t vgId) { - SVnodeObj *pVnode = NULL; - if (tsVnodesHash != NULL) { - taosHashGetClone(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, &pVnode); - } - - if (pVnode == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - vDebug("vgId:%d, not exist", vgId); - return NULL; - } - - return pVnode; -} - -void vnodeRelease(void *vparam) { - SVnodeObj *pVnode = vparam; - if (vparam == NULL) return; - - int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - int32_t vgId = pVnode->vgId; - - vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", vgId, refCount, pVnode); - assert(refCount >= 0); - - if (refCount > 0) { - if (vnodeInResetStatus(pVnode) && refCount <= 3) { - tsem_post(&pVnode->sem); - } - } else { - vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", vgId, refCount, pVnode); - vnodeDestroyInMWorker(pVnode); - int32_t count = taosHashGetSize(tsVnodesHash); - vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count); - } -} - -void *vnodeAcquireNotClose(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode != NULL && pVnode->preClose == 1) { - vnodeRelease(pVnode); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - vDebug("vgId:%d, not exist, pre closing", vgId); - return NULL; - } - - return pVnode; -} - -static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { - int64_t totalStorage = 0; - int64_t compStorage = 0; - int64_t pointsWritten = 0; - - if (vnodeInClosingStatus(pVnode)) return; - if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; - - if (pVnode->tsdb) { - tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage); - } - - SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; - pLoad->vgId = htonl(pVnode->vgId); - pLoad->dbCfgVersion = htonl(pVnode->dbCfgVersion); - pLoad->vgCfgVersion = htonl(pVnode->vgCfgVersion); - pLoad->totalStorage = htobe64(totalStorage); - pLoad->compStorage = htobe64(compStorage); - pLoad->pointsWritten = htobe64(pointsWritten); - pLoad->vnodeVersion = htobe64(pVnode->version); - pLoad->status = pVnode->status; - pLoad->role = pVnode->role; - pLoad->replica = pVnode->syncCfg.replica; - pLoad->compact = (pVnode->tsdb != NULL) ? tsdbGetCompactState(pVnode->tsdb) : 0; -} - -int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - void *pIter = taosHashIterate(tsVnodesHash, NULL); - while (pIter) { - SVnodeObj **pVnode = pIter; - if (*pVnode) { - - (*numOfVnodes)++; - if (*numOfVnodes >= TSDB_MAX_VNODES) { - vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES); - continue; - } else { - vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId; - } - - } - - pIter = taosHashIterate(tsVnodesHash, pIter); - } - return TSDB_CODE_SUCCESS; -} - -void vnodeBuildStatusMsg(void *param) { - SStatusMsg *pStatus = param; - - void *pIter = taosHashIterate(tsVnodesHash, NULL); - while (pIter) { - SVnodeObj **pVnode = pIter; - if (*pVnode) { - vnodeBuildVloadMsg(*pVnode, pStatus); - } - pIter = taosHashIterate(tsVnodesHash, pIter); - } -} - -void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { - for (int32_t i = 0; i < numOfVnodes; ++i) { - pAccess[i].vgId = htonl(pAccess[i].vgId); - SVnodeObj *pVnode = vnodeAcquireNotClose(pAccess[i].vgId); - if (pVnode != NULL) { - pVnode->accessState = pAccess[i].accessState; - if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) { - vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState); - } - vnodeRelease(pVnode); - } - } -} diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c deleted file mode 100644 index 2bdfd2ead3a31d8c2cba94d93239de965d2e07dc..0000000000000000000000000000000000000000 --- a/src/vnode/src/vnodeSync.c +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosmsg.h" -#include "query.h" -#include "dnode.h" -#include "vnodeVersion.h" -#include "vnodeMain.h" -#include "vnodeStatus.h" - -uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while get file info", vgId); - return 0; - } - - *fver = pVnode->fversion; - uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); - - vnodeRelease(pVnode); - return ret; -} - -int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while get wal info", vgId); - return -1; - } - - int32_t code = walGetWalFile(pVnode->wal, fileName, fileId); - - vnodeRelease(pVnode); - return code; -} - -void vnodeNotifyRole(int32_t vgId, int8_t role) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vTrace("vgId:%d, vnode not found while notify role", vgId); - return; - } - if (pVnode->dropped) { - vTrace("vgId:%d, vnode dropped while notify role", vgId); - vnodeRelease(pVnode); - return; - } - - vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); - pVnode->role = role; - dnodeSendStatusMsgToMnode(); - - if (pVnode->role == TAOS_SYNC_ROLE_MASTER) { - cqStart(pVnode->cq); - } else { - cqStop(pVnode->cq); - } - - vnodeRelease(pVnode); -} - -void vnodeCtrlFlow(int32_t vgId, int32_t level) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vTrace("vgId:%d, vnode not found while flow ctrl", vgId); - return; - } - if (pVnode->dropped) { - vTrace("vgId:%d, vnode dropped while flow ctrl", vgId); - vnodeRelease(pVnode); - return; - } - - if (pVnode->flowctrlLevel != level) { - vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); - pVnode->flowctrlLevel = level; - } - - vnodeRelease(pVnode); -} - -void vnodeStartSyncFile(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while start filesync", vgId); - return; - } - - vInfo("vgId:%d, datafile will be synced", vgId); - vnodeSetResetStatus(pVnode); - - vnodeRelease(pVnode); -} - -void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while stop filesync", vgId); - return; - } - - pVnode->fversion = fversion; - pVnode->version = fversion; - vnodeSaveVersion(pVnode); - walResetVersion(pVnode->wal, fversion); - - vInfo("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); - vnodeSetReadyStatus(pVnode); - - vnodeRelease(pVnode); -} - -void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while confirm forward", vgId); - } - - if (code == TSDB_CODE_SYN_CONFIRM_EXPIRED && pVnode->status == TAOS_VN_STATUS_CLOSING) { - vDebug("vgId:%d, db:%s, vnode is closing while confirm forward", vgId, pVnode->db); - code = TSDB_CODE_VND_IS_CLOSING; - } - - dnodeSendRpcVWriteRsp(pVnode, wparam, code); - vnodeRelease(pVnode); -} - -int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while write to cache", vgId); - vnodeRelease(pVnode); - return TSDB_CODE_VND_INVALID_VGROUP_ID; - } - - int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam); - - vnodeRelease(pVnode); - return code; -} - -int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { - SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while write to cache", vgId); - return -1; - } - - int32_t code = 0; - if (pVnode->isCommiting) { - vInfo("vgId:%d, vnode is commiting while get version", vgId); - code = -1; - } else { - *fver = pVnode->fversion; - *wver = pVnode->version; - } - - vnodeRelease(pVnode); - return code; -} - -void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code, bool force) { - SVnodeObj *pVnode = vparam; - syncConfirmForward(pVnode->sync, version, code, force); -} diff --git a/src/vnode/src/vnodeWorker.c b/src/vnode/src/vnodeWorker.c deleted file mode 100644 index 7fcc393746639777af20730f9daf8d7533c2b5e6..0000000000000000000000000000000000000000 --- a/src/vnode/src/vnodeWorker.c +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tutil.h" -#include "tqueue.h" -#include "tglobal.h" -#include "vnodeWorker.h" -#include "vnodeMain.h" - -typedef enum { - VNODE_WORKER_ACTION_CLEANUP, - VNODE_WORKER_ACTION_DESTROY -} EVMWorkerAction; - -typedef struct { - int32_t vgId; - int32_t code; - void * rpcHandle; - SVnodeObj *pVnode; - EVMWorkerAction action; -} SVMWorkerMsg; - -typedef struct { - pthread_t thread; - int32_t workerId; -} SVMWorker; - -typedef struct { - int32_t curNum; - int32_t maxNum; - SVMWorker *worker; -} SVMWorkerPool; - -static SVMWorkerPool tsVMWorkerPool; -static taos_qset tsVMWorkerQset; -static taos_queue tsVMWorkerQueue; - -static void *vnodeMWorkerFunc(void *param); - -static int32_t vnodeStartMWorker() { - tsVMWorkerQueue = taosOpenQueue(); - if (tsVMWorkerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; - - taosAddIntoQset(tsVMWorkerQset, tsVMWorkerQueue, NULL); - - for (int32_t i = tsVMWorkerPool.curNum; i < tsVMWorkerPool.maxNum; ++i) { - SVMWorker *pWorker = tsVMWorkerPool.worker + i; - pWorker->workerId = i; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&pWorker->thread, &thAttr, vnodeMWorkerFunc, pWorker) != 0) { - vError("failed to create thread to process vmworker queue, reason:%s", strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - - tsVMWorkerPool.curNum = i + 1; - vDebug("vmworker:%d is launched, total:%d", pWorker->workerId, tsVMWorkerPool.maxNum); - } - - vDebug("vmworker queue:%p is allocated", tsVMWorkerQueue); - return TSDB_CODE_SUCCESS; -} - -int32_t vnodeInitMWorker() { - tsVMWorkerQset = taosOpenQset(); - - tsVMWorkerPool.maxNum = 1; - tsVMWorkerPool.curNum = 0; - tsVMWorkerPool.worker = calloc(sizeof(SVMWorker), tsVMWorkerPool.maxNum); - - if (tsVMWorkerPool.worker == NULL) return -1; - for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { - SVMWorker *pWorker = tsVMWorkerPool.worker + i; - pWorker->workerId = i; - vDebug("vmworker:%d is created", i); - } - - vDebug("vmworker is initialized, num:%d qset:%p", tsVMWorkerPool.maxNum, tsVMWorkerQset); - - return vnodeStartMWorker(); -} - -static void vnodeStopMWorker() { - vDebug("vmworker queue:%p is freed", tsVMWorkerQueue); - taosCloseQueue(tsVMWorkerQueue); - tsVMWorkerQueue = NULL; -} - -void vnodeCleanupMWorker() { - for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { - SVMWorker *pWorker = tsVMWorkerPool.worker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - taosQsetThreadResume(tsVMWorkerQset); - } - vDebug("vmworker:%d is closed", i); - } - - for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { - SVMWorker *pWorker = tsVMWorkerPool.worker + i; - vDebug("vmworker:%d start to join", i); - if (taosCheckPthreadValid(pWorker->thread)) { - pthread_join(pWorker->thread, NULL); - } - vDebug("vmworker:%d join success", i); - } - - vDebug("vmworker is closed, qset:%p", tsVMWorkerQset); - - taosCloseQset(tsVMWorkerQset); - tsVMWorkerQset = NULL; - tfree(tsVMWorkerPool.worker); - - vnodeStopMWorker(); -} - -static int32_t vnodeWriteIntoMWorker(SVnodeObj *pVnode, EVMWorkerAction action, void *rpcHandle) { - SVMWorkerMsg *pMsg = taosAllocateQitem(sizeof(SVMWorkerMsg)); - if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY; - - pMsg->vgId = pVnode->vgId; - pMsg->pVnode = pVnode; - pMsg->rpcHandle = rpcHandle; - pMsg->action = action; - - int32_t code = taosWriteQitem(tsVMWorkerQueue, TAOS_QTYPE_RPC, pMsg); - if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS; - - return code; -} - -int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode) { - vTrace("vgId:%d, will cleanup in vmworker", pVnode->vgId); - return vnodeWriteIntoMWorker(pVnode, VNODE_WORKER_ACTION_CLEANUP, NULL); -} - -int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode) { - vTrace("vgId:%d, will destroy in vmworker", pVnode->vgId); - return vnodeWriteIntoMWorker(pVnode, VNODE_WORKER_ACTION_DESTROY, NULL); -} - -static void vnodeFreeMWorkerMsg(SVMWorkerMsg *pMsg) { - vTrace("vgId:%d, disposed in vmworker", pMsg->vgId); - taosFreeQitem(pMsg); -} - -static void vnodeSendVMWorkerRpcRsp(SVMWorkerMsg *pMsg) { - if (pMsg->rpcHandle != NULL) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcHandle, .code = pMsg->code}; - rpcSendResponse(&rpcRsp); - } - - vnodeFreeMWorkerMsg(pMsg); -} - -static void vnodeProcessMWorkerMsg(SVMWorkerMsg *pMsg) { - pMsg->code = 0; - - switch (pMsg->action) { - case VNODE_WORKER_ACTION_CLEANUP: - vnodeCleanUp(pMsg->pVnode); - break; - case VNODE_WORKER_ACTION_DESTROY: - vnodeDestroy(pMsg->pVnode); - break; - default: - break; - } -} - -static void *vnodeMWorkerFunc(void *param) { - setThreadName("vnodeMWorker"); - - while (1) { - SVMWorkerMsg *pMsg = NULL; - if (taosReadQitemFromQset(tsVMWorkerQset, NULL, (void **)&pMsg, NULL) == 0) { - vDebug("qset:%p, vmworker got no message from qset, exiting", tsVMWorkerQset); - break; - } - - vTrace("vgId:%d, action:%d will be processed in vmworker queue", pMsg->vgId, pMsg->action); - vnodeProcessMWorkerMsg(pMsg); - vnodeSendVMWorkerRpcRsp(pMsg); - } - - return NULL; -} diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c deleted file mode 100644 index 743398d8344b8430a71633fe2455bca4e5ae1682..0000000000000000000000000000000000000000 --- a/src/vnode/src/vnodeWrite.c +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tp.h" -#include "taosmsg.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tqueue.h" -#include "ttimer.h" -#include "dnode.h" -#include "vnodeStatus.h" - -#define MAX_QUEUED_MSG_NUM 100000 -#define MAX_QUEUED_MSG_SIZE 1024*1024*1024 //1GB - -extern void * tsDnodeTmr; -static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, SRspRet *); -static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); -static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); -static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); -static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); -static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); -static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); -static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite); - -int32_t vnodeInitWrite(void) { - vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessSubmitMsg; - vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessCreateTableMsg; - vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = vnodeProcessDropTableMsg; - vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = vnodeProcessAlterTableMsg; - vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessDropStableMsg; - vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; - - return 0; -} - -void vnodeCleanupWrite() {} - -int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { - int32_t code = 0; - SVnodeObj *pVnode = vparam; - SWalHead * pHead = wparam; - SVWriteMsg*pWrite = rparam; - - SRspRet *pRspRet = NULL; - if (pWrite != NULL) pRspRet = &pWrite->rspRet; - - if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) { - vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId, - taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version); - return TSDB_CODE_VND_MSG_NOT_PROCESSED; - } - - vTrace("vgId:%d, msg:%s will be processed in vnode, qtype:%s hver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, - taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version, pVnode->version); - - if (pHead->version == 0) { // from client or CQ - if (!vnodeInReadyStatus(pVnode)) { - vDebug("vgId:%d, msg:%s not processed since vstatus:%d, qtype:%s hver:%" PRIu64, pVnode->vgId, - taosMsg[pHead->msgType], pVnode->status, qtypeStr[qtype], pHead->version); - return TSDB_CODE_APP_NOT_READY; // it may be in deleting or closing state - } - - if (pVnode->role != TAOS_SYNC_ROLE_MASTER) { - vDebug("vgId:%d, msg:%s not processed since replica:%d role:%s, qtype:%s hver:%" PRIu64, pVnode->vgId, - taosMsg[pHead->msgType], pVnode->syncCfg.replica, syncRole[pVnode->role], qtypeStr[qtype], pHead->version); - return TSDB_CODE_APP_NOT_READY; - } - - // assign version - pHead->version = pVnode->version + 1; - } else { // from wal or forward - // for data from WAL or forward, version may be smaller - if (pHead->version <= pVnode->version) return 0; - } - - // forward to peers, even it is WAL/FWD, it shall be called to update version in sync - int32_t syncCode = 0; - bool force = (pWrite == NULL ? false : pWrite->walHead.msgType != TSDB_MSG_TYPE_SUBMIT); - syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype, force); - if (syncCode < 0) { - pHead->version = 0; - return syncCode; - } - - // write into WAL - code = walWrite(pVnode->wal, pHead); - if (code < 0) { - if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1); - vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code); - pHead->version = 0; - return code; - } - - pVnode->version = pHead->version; - - // write data locally - code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet); - if (code < 0) { - if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1); - return code; - } - - return syncCode; -} - -static int32_t vnodeCheckWrite(SVnodeObj *pVnode) { - if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { - vDebug("vgId:%d, no write auth, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - return TSDB_CODE_VND_NO_WRITE_AUTH; - } - - if (pVnode->dbReplica != pVnode->syncCfg.replica && - pVnode->syncCfg.nodeInfo[pVnode->syncCfg.replica - 1].nodeId == dnodeGetDnodeId()) { - vDebug("vgId:%d, vnode is balancing and will be dropped, dbReplica:%d vgReplica:%d, refCount:%d pVnode:%p", - pVnode->vgId, pVnode->dbReplica, pVnode->syncCfg.replica, pVnode->refCount, pVnode); - return TSDB_CODE_VND_IS_BALANCING; - } - - // tsdb may be in reset state - if (pVnode->tsdb == NULL) { - vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; - } - - if (pVnode->isFull) { - vDebug("vgId:%d, vnode is full, refCount:%d", pVnode->vgId, pVnode->refCount); - return TSDB_CODE_VND_IS_FULL; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - int32_t code = TSDB_CODE_SUCCESS; - - vTrace("vgId:%d, submit msg is processed", pVnode->vgId); - - if (pVnode->dbType == TSDB_DB_TYPE_TOPIC && pVnode->role == TAOS_SYNC_ROLE_MASTER) { - tpUpdateTs(pVnode->vgId, &pVnode->sequence, pCont); - } - - // save insert result into item - SShellSubmitRspMsg *pRsp = NULL; - if (pRet) { - pRet->len = sizeof(SShellSubmitRspMsg); - pRet->rsp = rpcMallocCont(pRet->len); - pRsp = pRet->rsp; - } - - if (tsdbInsertData(pVnode->tsdb, pCont, pRsp) < 0) code = terrno; - - return code; -} - -static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - int code = TSDB_CODE_SUCCESS; - - STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont); - if (pCfg == NULL) { - ASSERT(terrno != 0); - return terrno; - } - - if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) { - code = terrno; - ASSERT(code != 0); - } - - tsdbClearTableCfg(pCfg); - return code; -} - -static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - SMDDropTableMsg *pTable = pCont; - int32_t code = TSDB_CODE_SUCCESS; - - vDebug("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableFname); - STableId tableId = {.uid = htobe64(pTable->uid), .tid = htonl(pTable->tid)}; - - if (tsdbDropTable(pVnode->tsdb, tableId) < 0) code = terrno; - - return code; -} - -static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - // TODO: disposed in tsdb - // STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont); - // if (pCfg == NULL) return terrno; - // if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno; - - // tsdbClearTableCfg(pCfg); - vDebug("vgId:%d, alter table msg is received", pVnode->vgId); - return TSDB_CODE_SUCCESS; -} - -static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - SDropSTableMsg *pTable = pCont; - int32_t code = TSDB_CODE_SUCCESS; - - vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableFname); - - STableId stableId = {.uid = htobe64(pTable->uid), .tid = -1}; - - if (tsdbDropTable(pVnode->tsdb, stableId) < 0) code = terrno; - - vDebug("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableFname, tstrerror(code)); - - return code; -} - -static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - if (tsdbUpdateTableTagValue(pVnode->tsdb, (SUpdateTableTagValMsg *)pCont) < 0) { - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -static SVWriteMsg *vnodeBuildVWriteMsg(SVnodeObj *pVnode, SWalHead *pHead, int32_t qtype, SRpcMsg *pRpcMsg) { - if (pHead->len > TSDB_MAX_WAL_SIZE) { - vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version); - terrno = TSDB_CODE_WAL_SIZE_LIMIT; - return NULL; - } - - int32_t size = sizeof(SVWriteMsg) + pHead->len; - SVWriteMsg *pWrite = taosAllocateQitem(size); - if (pWrite == NULL) { - terrno = TSDB_CODE_VND_OUT_OF_MEMORY; - return NULL; - } - - if (pRpcMsg != NULL) { - pWrite->rpcMsg = *pRpcMsg; - } - - memcpy(&pWrite->walHead, pHead, sizeof(SWalHead) + pHead->len); - pWrite->pVnode = pVnode; - pWrite->qtype = qtype; - - atomic_add_fetch_32(&pVnode->refCount, 1); - - return pWrite; -} - -static int32_t vnodeWriteToWQueueImp(SVWriteMsg *pWrite) { - SVnodeObj *pVnode = pWrite->pVnode; - - if (pWrite->qtype == TAOS_QTYPE_RPC) { - int32_t code = vnodeCheckWrite(pVnode); - if (code != TSDB_CODE_SUCCESS) { - vError("vgId:%d, failed to write into vwqueue since %s", pVnode->vgId, tstrerror(code)); - taosFreeQitem(pWrite); - vnodeRelease(pVnode); - return code; - } - } - - if (tsAvailDataDirGB <= tsMinimalDataDirGB) { - vError("vgId:%d, failed to write into vwqueue since no diskspace, avail:%fGB", pVnode->vgId, tsAvailDataDirGB); - taosFreeQitem(pWrite); - vnodeRelease(pVnode); - return TSDB_CODE_VND_NO_DISKSPACE; - } - - if (!vnodeInReadyOrUpdatingStatus(pVnode)) { - vError("vgId:%d, failed to write into vwqueue, vstatus is %s, refCount:%d pVnode:%p", pVnode->vgId, - vnodeStatus[pVnode->status], pVnode->refCount, pVnode); - taosFreeQitem(pWrite); - vnodeRelease(pVnode); - return TSDB_CODE_APP_NOT_READY; - } - - int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); - int64_t queuedSize = atomic_add_fetch_64(&pVnode->queuedWMsgSize, pWrite->walHead.len); - - if (queued > MAX_QUEUED_MSG_NUM || queuedSize > MAX_QUEUED_MSG_SIZE) { - int32_t ms = (queued / MAX_QUEUED_MSG_NUM) * 10 + 3; - if (ms > 100) ms = 100; - vDebug("vgId:%d, too many msg:%d in vwqueue, flow control %dms", pVnode->vgId, queued, ms); - taosMsleep(ms); - } - - vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d size:%" PRId64, pVnode->vgId, pVnode->refCount, - pVnode->queuedWMsg, pVnode->queuedWMsgSize); - - taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); - return TSDB_CODE_SUCCESS; -} - -int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { - SVnodeObj *pVnode = vparam; - if (qtype == TAOS_QTYPE_RPC) { - if (!vnodeInReadyStatus(pVnode)) { - return TSDB_CODE_APP_NOT_READY; // it may be in deleting or closing state - } - - if (pVnode->role != TAOS_SYNC_ROLE_MASTER) { - return TSDB_CODE_APP_NOT_READY; - } - } - - SVWriteMsg *pWrite = vnodeBuildVWriteMsg(vparam, wparam, qtype, rparam); - if (pWrite == NULL) { - assert(terrno != 0); - return terrno; - } - - int32_t code = vnodePerformFlowCtrl(pWrite); - if (code != 0) return 0; - - return vnodeWriteToWQueueImp(pWrite); -} - -void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { - SVnodeObj *pVnode = vparam; - if (pVnode) { - int32_t queued = atomic_sub_fetch_32(&pVnode->queuedWMsg, 1); - int64_t queuedSize = atomic_sub_fetch_64(&pVnode->queuedWMsgSize, pWrite->walHead.len); - - vTrace("vgId:%d, msg:%p, app:%p, free from vwqueue, queued:%d size:%" PRId64, pVnode->vgId, pWrite, - pWrite->rpcMsg.ahandle, queued, queuedSize); - } - - taosFreeQitem(pWrite); - vnodeRelease(pVnode); -} - -static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { - SVWriteMsg *pWrite = param; - SVnodeObj * pVnode = pWrite->pVnode; - int32_t code = TSDB_CODE_VND_IS_SYNCING; - - if (pVnode->flowctrlLevel <= 0) code = TSDB_CODE_VND_IS_FLOWCTRL; - - pWrite->processedCount++; - if (pWrite->processedCount >= 100) { - vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), - pWrite->processedCount); - void *handle = pWrite->rpcMsg.handle; - taosFreeQitem(pWrite); - vnodeRelease(pVnode); - SRpcMsg rpcRsp = {.handle = handle, .code = code}; - rpcSendResponse(&rpcRsp); - } else { - code = vnodePerformFlowCtrl(pWrite); - if (code == 0) { - vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite, - pWrite->processedCount); - pWrite->processedCount = 0; - void *handle = pWrite->rpcMsg.handle; - code = vnodeWriteToWQueueImp(pWrite); - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rpcRsp = {.handle = handle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } - } -} - -static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { - SVnodeObj *pVnode = pWrite->pVnode; - if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; - if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->queuedWMsgSize < MAX_QUEUED_MSG_SIZE && - pVnode->flowctrlLevel <= 0) - return 0; - - if (tsEnableFlowCtrl == 0) { - int32_t ms = (int32_t)pow(2, pVnode->flowctrlLevel + 2); - if (ms > 100) ms = 100; - vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); - taosMsleep(ms); - return 0; - } else { - void *unUsedTimerId = NULL; - taosTmrReset(vnodeFlowCtrlMsgToWQueue, 100, pWrite, tsDnodeTmr, &unUsedTimerId); - - vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, - pWrite->processedCount); - return TSDB_CODE_VND_ACTION_IN_PROGRESS; - } -} - -void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { - int32_t extraSleep = 0; - while (pVnode->queuedWMsg > 0) { - vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); - taosMsleep(10); - extraSleep = 1; - } - - if (extraSleep) - taosMsleep(900); -}