Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 68f7c0ec 编写于 作者: H Haojun Liao
浏览文件
操作

Merge remote-tracking branch 'origin/feature/vnode_refact1' into feature/3.0_liaohj

上级 d62ddcf2 3e6172c6
展开全部 隐藏空白更改
内联 并排
Showing with 1530 addition and 4454 deletion
.devcontainer/Dockerfile
浏览文件 @ 68f7c0ec
......@@ -7,4 +7,4 @@ FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
# [Optional] Uncomment this section to install additional packages.
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# && apt-get -y install --no-install-recommends <your-package-list-here>
RUN apt-get update && apt-get -y install tree vim tmux
RUN apt-get update && apt-get -y install tree vim tmux python3-pip
contrib/CMakeLists.txt
浏览文件 @ 68f7c0ec
......@@ -110,7 +110,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
# ================================================================================================
# googletest
if(${BUILD_TEST})
add_subdirectory(googletest)
add_subdirectory(googletest EXCLUDE_FROM_ALL)
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
......@@ -143,7 +143,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
add_subdirectory(cJson)
add_subdirectory(cJson EXCLUDE_FROM_ALL)
target_include_directories(
cjson
# see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
......@@ -152,7 +152,7 @@ target_include_directories(
unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# lz4
add_subdirectory(lz4/build/cmake)
add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
target_include_directories(
lz4_static
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
......@@ -160,7 +160,7 @@ target_include_directories(
# zlib
set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
add_subdirectory(zlib)
add_subdirectory(zlib EXCLUDE_FROM_ALL)
target_include_directories(
zlibstatic
PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
......@@ -176,7 +176,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# leveldb
if(${BUILD_WITH_LEVELDB})
option(LEVELDB_BUILD_TESTS "" OFF)
add_subdirectory(leveldb)
add_subdirectory(leveldb EXCLUDE_FROM_ALL)
target_include_directories(
leveldb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
......@@ -192,7 +192,7 @@ if(${BUILD_WITH_ROCKSDB})
option(WITH_TOOLS "" OFF)
option(WITH_LIBURING "" OFF)
option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
add_subdirectory(rocksdb)
add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
target_include_directories(
rocksdb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
......@@ -203,7 +203,7 @@ endif(${BUILD_WITH_ROCKSDB})
# To support build on ubuntu: sudo apt-get install libboost-all-dev
if(${BUILD_WITH_LUCENE})
option(ENABLE_TEST "Enable the tests" OFF)
add_subdirectory(lucene)
add_subdirectory(lucene EXCLUDE_FROM_ALL)
target_include_directories(
lucene++
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
......@@ -213,14 +213,14 @@ endif(${BUILD_WITH_LUCENE})
# NuRaft
if(${BUILD_WITH_NURAFT})
add_subdirectory(nuraft)
add_subdirectory(nuraft EXCLUDE_FROM_ALL)
endif(${BUILD_WITH_NURAFT})
# pthread
if(${BUILD_PTHREAD})
set(CMAKE_BUILD_TYPE release)
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
add_subdirectory(pthread EXCLUDE_FROM_ALL)
set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
add_library(pthread STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
......@@ -228,12 +228,12 @@ endif()
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv)
add_subdirectory(iconv EXCLUDE_FROM_ALL)
endif(${BUILD_WITH_ICONV})
# wingetopt
if(${BUILD_WINGETOPT})
add_subdirectory(wingetopt)
add_subdirectory(wingetopt EXCLUDE_FROM_ALL)
endif(${BUILD_WINGETOPT})
# msvcregex
......@@ -293,7 +293,7 @@ if(${BUILD_WITH_UV})
MESSAGE("Windows need set no-sign-compare")
add_compile_options(-Wno-sign-compare)
endif ()
add_subdirectory(libuv)
add_subdirectory(libuv EXCLUDE_FROM_ALL)
endif(${BUILD_WITH_UV})
# BDB
......@@ -334,5 +334,5 @@ endif(${BUILD_WITH_SQLITE})
# Build test
# ================================================================================================
if(${BUILD_DEPENDENCY_TESTS})
add_subdirectory(test)
add_subdirectory(test EXCLUDE_FROM_ALL)
endif(${BUILD_DEPENDENCY_TESTS})
include/common/tmsg.h
浏览文件 @ 68f7c0ec
......@@ -177,6 +177,7 @@ typedef struct SField {
char name[TSDB_COL_NAME_LEN];
uint8_t type;
int32_t bytes;
int8_t flags;
} SField;
typedef struct SRetention {
......@@ -302,13 +303,11 @@ typedef struct {
int32_t ttl;
int32_t numOfColumns;
int32_t numOfTags;
int32_t numOfSmas;
int32_t commentLen;
int32_t ast1Len;
int32_t ast2Len;
SArray* pColumns; // array of SField
SArray* pTags; // array of SField
SArray* pSmas; // array of SField
char* comment;
char* pAst1;
char* pAst2;
......@@ -2384,9 +2383,9 @@ typedef struct {
int32_t epoch;
uint64_t reqId;
int64_t consumerId;
int64_t blockingTime;
int64_t waitTime;
int64_t currentOffset;
} SMqPollReqV2;
} SMqPollReq;
typedef struct {
int32_t vgId;
......@@ -2401,53 +2400,6 @@ typedef struct {
SSchemaWrapper schema;
} SMqSubTopicEp;
typedef struct {
SMqRspHead head;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
int32_t dataLen;
SArray* blockPos; // beginning pos for each SRetrieveTableRsp
void* blockData; // serialized batched SRetrieveTableRsp
} SMqPollRspV2;
static FORCE_INLINE int32_t tEncodeSMqPollRspV2(void** buf, const SMqPollRspV2* pRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
tlen += taosEncodeFixedI32(buf, pRsp->dataLen);
if (pRsp->dataLen != 0) {
int32_t sz = taosArrayGetSize(pRsp->blockPos);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
int32_t blockPos = *(int32_t*)taosArrayGet(pRsp->blockPos, i);
tlen += taosEncodeFixedI32(buf, blockPos);
}
tlen += taosEncodeBinary(buf, pRsp->blockData, pRsp->dataLen);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRsp) {
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
buf = taosDecodeFixedI32(buf, &pRsp->dataLen);
if (pRsp->dataLen != 0) {
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pRsp->blockPos = taosArrayInit(sz, sizeof(int32_t));
for (int32_t i = 0; i < sz; i++) {
int32_t blockPos;
buf = taosDecodeFixedI32(buf, &blockPos);
taosArrayPush(pRsp->blockPos, &blockPos);
}
buf = taosDecodeBinary(buf, &pRsp->blockData, pRsp->dataLen);
}
return (void*)buf;
}
typedef struct {
SMqRspHead head;
int64_t reqOffset;
......
include/dnode/mnode/sdb/sdb.h
浏览文件 @ 68f7c0ec
......@@ -18,6 +18,11 @@
#include "os.h"
#include "thash.h"
#include "tlockfree.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
......@@ -135,7 +140,7 @@ typedef enum {
typedef struct SSdb SSdb;
typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj);
typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
......@@ -326,9 +331,29 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
int32_t sdbGetRawTotalSize(SSdbRaw *pRaw);
SSdbRow *sdbAllocRow(int32_t objSize);
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow);
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
void *sdbGetRowObj(SSdbRow *pRow);
typedef struct SSdb {
SMnode *pMnode;
char *currDir;
char *syncDir;
char *tmpDir;
int64_t lastCommitVer;
int64_t curVer;
int64_t tableVer[SDB_MAX];
int64_t maxId[SDB_MAX];
EKeyType keyTypes[SDB_MAX];
SHashObj *hashObjs[SDB_MAX];
SRWLatch locks[SDB_MAX];
SdbInsertFp insertFps[SDB_MAX];
SdbUpdateFp updateFps[SDB_MAX];
SdbDeleteFp deleteFps[SDB_MAX];
SdbDeployFp deployFps[SDB_MAX];
SdbEncodeFp encodeFps[SDB_MAX];
SdbDecodeFp decodeFps[SDB_MAX];
} SSdb;
#ifdef __cplusplus
}
#endif
......
include/libs/function/functionMgt.h
浏览文件 @ 68f7c0ec
......@@ -20,8 +20,8 @@
extern "C" {
#endif
#include "querynodes.h"
#include "function.h"
#include "querynodes.h"
typedef enum EFunctionType {
// aggregate function
......@@ -123,10 +123,10 @@ struct SCatalog;
typedef struct SFmGetFuncInfoParam {
struct SCatalog* pCtg;
void *pRpc;
const SEpSet* pMgmtEps;
char* pErrBuf;
int32_t errBufLen;
void* pRpc;
const SEpSet* pMgmtEps;
char* pErrBuf;
int32_t errBufLen;
} SFmGetFuncInfoParam;
int32_t fmFuncMgtInit();
......@@ -143,6 +143,7 @@ bool fmIsDatetimeFunc(int32_t funcId);
bool fmIsTimelineFunc(int32_t funcId);
bool fmIsTimeorderFunc(int32_t funcId);
bool fmIsPseudoColumnFunc(int32_t funcId);
bool fmIsScanPseudoColumnFunc(int32_t funcId);
bool fmIsWindowPseudoColumnFunc(int32_t funcId);
bool fmIsWindowClauseFunc(int32_t funcId);
bool fmIsSpecialDataRequiredFunc(int32_t funcId);
......
include/libs/nodes/plannodes.h
浏览文件 @ 68f7c0ec
......@@ -20,50 +20,46 @@
extern "C" {
#endif
#include "querynodes.h"
#include "query.h"
#include "querynodes.h"
#include "tname.h"
typedef struct SLogicNode {
ENodeType type;
SNodeList* pTargets; // SColumnNode
SNode* pConditions;
SNodeList* pChildren;
ENodeType type;
SNodeList* pTargets; // SColumnNode
SNode* pConditions;
SNodeList* pChildren;
struct SLogicNode* pParent;
int32_t optimizedFlag;
int32_t optimizedFlag;
} SLogicNode;
typedef enum EScanType {
SCAN_TYPE_TAG,
SCAN_TYPE_TABLE,
SCAN_TYPE_SYSTEM_TABLE,
SCAN_TYPE_STREAM
} EScanType;
typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType;
typedef struct SScanLogicNode {
SLogicNode node;
SNodeList* pScanCols;
SLogicNode node;
SNodeList* pScanCols;
SNodeList* pScanPseudoCols;
struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList;
EScanType scanType;
uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange;
SName tableName;
bool showRewrite;
double ratio;
SNodeList* pDynamicScanFuncs;
int32_t dataRequired;
int64_t interval;
int64_t offset;
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
SVgroupsInfo* pVgroupList;
EScanType scanType;
uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange;
SName tableName;
bool showRewrite;
double ratio;
SNodeList* pDynamicScanFuncs;
int32_t dataRequired;
int64_t interval;
int64_t offset;
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
} SScanLogicNode;
typedef struct SJoinLogicNode {
SLogicNode node;
EJoinType joinType;
SNode* pOnConditions;
EJoinType joinType;
SNode* pOnConditions;
} SJoinLogicNode;
typedef struct SAggLogicNode {
......@@ -75,47 +71,43 @@ typedef struct SAggLogicNode {
typedef struct SProjectLogicNode {
SLogicNode node;
SNodeList* pProjections;
char stmtName[TSDB_TABLE_NAME_LEN];
int64_t limit;
int64_t offset;
int64_t slimit;
int64_t soffset;
char stmtName[TSDB_TABLE_NAME_LEN];
int64_t limit;
int64_t offset;
int64_t slimit;
int64_t soffset;
} SProjectLogicNode;
typedef struct SVnodeModifLogicNode {
SLogicNode node;
int32_t msgType;
SArray* pDataBlocks;
SLogicNode node;
int32_t msgType;
SArray* pDataBlocks;
SVgDataBlocks* pVgDataBlocks;
} SVnodeModifLogicNode;
typedef struct SExchangeLogicNode {
SLogicNode node;
int32_t srcGroupId;
uint8_t precision;
int32_t srcGroupId;
uint8_t precision;
} SExchangeLogicNode;
typedef enum EWindowType {
WINDOW_TYPE_INTERVAL = 1,
WINDOW_TYPE_SESSION,
WINDOW_TYPE_STATE
} EWindowType;
typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
typedef struct SWindowLogicNode {
SLogicNode node;
SLogicNode node;
EWindowType winType;
SNodeList* pFuncs;
int64_t interval;
int64_t offset;
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
SFillNode* pFill;
int64_t sessionGap;
SNode* pTspk;
SNode* pStateExpr;
int8_t triggerType;
int64_t watermark;
SNodeList* pFuncs;
int64_t interval;
int64_t offset;
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
SFillNode* pFill;
int64_t sessionGap;
SNode* pTspk;
SNode* pStateExpr;
int8_t triggerType;
int64_t watermark;
} SWindowLogicNode;
typedef struct SSortLogicNode {
......@@ -137,59 +129,60 @@ typedef enum ESubplanType {
typedef struct SSubplanId {
uint64_t queryId;
int32_t groupId;
int32_t subplanId;
int32_t groupId;
int32_t subplanId;
} SSubplanId;
typedef struct SLogicSubplan {
ENodeType type;
SSubplanId id;
SNodeList* pChildren;
SNodeList* pParents;
SLogicNode* pNode;
ESubplanType subplanType;
ENodeType type;
SSubplanId id;
SNodeList* pChildren;
SNodeList* pParents;
SLogicNode* pNode;
ESubplanType subplanType;
SVgroupsInfo* pVgroupList;
int32_t level;
int32_t splitFlag;
int32_t level;
int32_t splitFlag;
} SLogicSubplan;
typedef struct SQueryLogicPlan {
ENodeType type;
ENodeType type;
SNodeList* pTopSubplans;
} SQueryLogicPlan;
typedef struct SSlotDescNode {
ENodeType type;
int16_t slotId;
int16_t slotId;
SDataType dataType;
bool reserve;
bool output;
bool tag;
bool reserve;
bool output;
bool tag;
} SSlotDescNode;
typedef struct SDataBlockDescNode {
ENodeType type;
int16_t dataBlockId;
ENodeType type;
int16_t dataBlockId;
SNodeList* pSlots;
int32_t totalRowSize;
int32_t outputRowSize;
uint8_t precision;
int32_t totalRowSize;
int32_t outputRowSize;
uint8_t precision;
} SDataBlockDescNode;
typedef struct SPhysiNode {
ENodeType type;
ENodeType type;
SDataBlockDescNode* pOutputDataBlockDesc;
SNode* pConditions;
SNodeList* pChildren;
struct SPhysiNode* pParent;
SNode* pConditions;
SNodeList* pChildren;
struct SPhysiNode* pParent;
} SPhysiNode;
typedef struct SScanPhysiNode {
SPhysiNode node;
SNodeList* pScanCols;
uint64_t uid; // unique id of the table
int8_t tableType;
SName tableName;
SNodeList* pScanPseudoCols;
uint64_t uid; // unique id of the table
int8_t tableType;
SName tableName;
} SScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode;
......@@ -197,23 +190,23 @@ typedef SScanPhysiNode SStreamScanPhysiNode;
typedef struct SSystemTableScanPhysiNode {
SScanPhysiNode scan;
SEpSet mgmtEpSet;
bool showRewrite;
int32_t accountId;
SEpSet mgmtEpSet;
bool showRewrite;
int32_t accountId;
} SSystemTableScanPhysiNode;
typedef struct STableScanPhysiNode {
SScanPhysiNode scan;
uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange;
double ratio;
int32_t dataRequired;
SNodeList* pDynamicScanFuncs;
int64_t interval;
int64_t offset;
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange;
double ratio;
int32_t dataRequired;
SNodeList* pDynamicScanFuncs;
int64_t interval;
int64_t offset;
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
} STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode;
......@@ -221,89 +214,89 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode;
typedef struct SProjectPhysiNode {
SPhysiNode node;
SNodeList* pProjections;
int64_t limit;
int64_t offset;
int64_t slimit;
int64_t soffset;
int64_t limit;
int64_t offset;
int64_t slimit;
int64_t soffset;
} SProjectPhysiNode;
typedef struct SJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNode* pOnConditions; // in or out tuple ?
EJoinType joinType;
SNode* pOnConditions; // in or out tuple ?
SNodeList* pTargets;
} SJoinPhysiNode;
typedef struct SAggPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function
SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function
SNodeList* pGroupKeys;
SNodeList* pAggFuncs;
} SAggPhysiNode;
typedef struct SDownstreamSourceNode {
ENodeType type;
ENodeType type;
SQueryNodeAddr addr;
uint64_t taskId;
uint64_t schedId;
uint64_t taskId;
uint64_t schedId;
} SDownstreamSourceNode;
typedef struct SExchangePhysiNode {
SPhysiNode node;
int32_t srcGroupId; // group id of datasource suplans
int32_t srcGroupId; // group id of datasource suplans
SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode
} SExchangePhysiNode;
typedef struct SWinodwPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of parameter expression of function
SNodeList* pExprs; // these are expression list of parameter expression of function
SNodeList* pFuncs;
SNode* pTspk; // timestamp primary key
int8_t triggerType;
int64_t watermark;
SNode* pTspk; // timestamp primary key
int8_t triggerType;
int64_t watermark;
} SWinodwPhysiNode;
typedef struct SIntervalPhysiNode {
SWinodwPhysiNode window;
int64_t interval;
int64_t offset;
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
SFillNode* pFill;
int64_t interval;
int64_t offset;
int64_t sliding;
int8_t intervalUnit;
int8_t slidingUnit;
SFillNode* pFill;
} SIntervalPhysiNode;
typedef struct SMultiTableIntervalPhysiNode {
SIntervalPhysiNode interval;
SNodeList* pPartitionKeys;
SNodeList* pPartitionKeys;
} SMultiTableIntervalPhysiNode;
typedef struct SSessionWinodwPhysiNode {
SWinodwPhysiNode window;
int64_t gap;
int64_t gap;
} SSessionWinodwPhysiNode;
typedef struct SStateWinodwPhysiNode {
SWinodwPhysiNode window;
SNode* pStateKey;
SNode* pStateKey;
} SStateWinodwPhysiNode;
typedef struct SSortPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function
SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function
SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
SNodeList* pTargets;
} SSortPhysiNode;
typedef struct SPartitionPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of partition_by_clause
SNodeList* pExprs; // these are expression list of partition_by_clause
SNodeList* pPartitionKeys;
SNodeList* pTargets;
} SPartitionPhysiNode;
typedef struct SDataSinkNode {
ENodeType type;
ENodeType type;
SDataBlockDescNode* pInputDataBlockDesc;
} SDataSinkNode;
......@@ -313,45 +306,41 @@ typedef struct SDataDispatcherNode {
typedef struct SDataInserterNode {
SDataSinkNode sink;
int32_t numOfTables;
uint32_t size;
char *pData;
int32_t numOfTables;
uint32_t size;
char* pData;
} SDataInserterNode;
typedef struct SSubplan {
ENodeType type;
SSubplanId id; // unique id of the subplan
ESubplanType subplanType;
int32_t msgType; // message type for subplan, used to denote the send message type to vnode.
int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner.
char dbFName[TSDB_DB_FNAME_LEN];
SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node
SQueryNodeStat execNodeStat; // only for scan subplan
SNodeList* pChildren; // the datasource subplan,from which to fetch the result
SNodeList* pParents; // the data destination subplan, get data from current subplan
SPhysiNode* pNode; // physical plan of current subplan
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
ENodeType type;
SSubplanId id; // unique id of the subplan
ESubplanType subplanType;
int32_t msgType; // message type for subplan, used to denote the send message type to vnode.
int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner.
char dbFName[TSDB_DB_FNAME_LEN];
SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node
SQueryNodeStat execNodeStat; // only for scan subplan
SNodeList* pChildren; // the datasource subplan,from which to fetch the result
SNodeList* pParents; // the data destination subplan, get data from current subplan
SPhysiNode* pNode; // physical plan of current subplan
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
} SSubplan;
typedef enum EExplainMode {
EXPLAIN_MODE_DISABLE = 1,
EXPLAIN_MODE_STATIC,
EXPLAIN_MODE_ANALYZE
} EExplainMode;
typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
typedef struct SExplainInfo {
EExplainMode mode;
bool verbose;
double ratio;
bool verbose;
double ratio;
} SExplainInfo;
typedef struct SQueryPlan {
ENodeType type;
uint64_t queryId;
int32_t numOfSubplans;
SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
ENodeType type;
uint64_t queryId;
int32_t numOfSubplans;
SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
SExplainInfo explainInfo;
SNodeList* pPlaceholderValues;
SNodeList* pPlaceholderValues;
} SQueryPlan;
void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);
......
include/libs/nodes/querynodes.h
浏览文件 @ 68f7c0ec
......@@ -293,7 +293,10 @@ typedef struct SExplainStmt {
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols);
typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId);
int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs);
......
include/libs/transport/trpc.h
浏览文件 @ 68f7c0ec
......@@ -59,9 +59,13 @@ typedef struct {
void * pNode;
} SNodeMsg;
typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf);
typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
///
// // SRpcMsg code
// REDIERE,
// NOT READY, EpSet
typedef bool (*RpcRfp)(int32_t code);
typedef struct SRpcInit {
uint16_t localPort; // local port
......
include/libs/wal/wal.h
浏览文件 @ 68f7c0ec
......@@ -72,8 +72,6 @@ extern "C" {
#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_MAGIC 0xFAFBFCFDULL
#define WAL_CUR_FAILED 1
#pragma pack(push, 1)
typedef enum {
TAOS_WAL_NOLOG = 0,
......
source/client/src/clientImpl.c
浏览文件 @ 68f7c0ec
......@@ -246,6 +246,12 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
pResInfo->numOfCols = numOfCols;
// TODO handle memory leak
if (pResInfo->fields != NULL) {
taosMemoryFree(pResInfo->fields);
}
if (pResInfo->userFields != NULL) {
taosMemoryFree(pResInfo->userFields);
}
pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
......
source/client/src/tmq.c
浏览文件 @ 68f7c0ec
......@@ -16,7 +16,6 @@
#include "clientInt.h"
#include "clientLog.h"
#include "parser.h"
#include "planner.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tglobal.h"
......@@ -68,7 +67,7 @@ struct tmq_conf_t {
char* user;
char* pass;
char* db;
tmq_commit_cb* commit_cb;
tmq_commit_cb* commitCb;
};
struct tmq_t {
......@@ -115,8 +114,8 @@ enum {
enum {
TMQ_CONSUMER_STATUS__INIT = 0,
TMQ_CONSUMER_STATUS__SUBSCRIBED,
TMQ_CONSUMER_STATUS__READY,
TMQ_CONSUMER_STATUS__NO_TOPIC,
};
enum {
......@@ -175,7 +174,6 @@ typedef struct {
int32_t epoch;
int32_t vgId;
tsem_t rspSem;
int32_t sync;
} SMqPollCbParam;
typedef struct {
......@@ -300,6 +298,7 @@ void tmqAssignDelayedHbTask(void* param, void* tmrId) {
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
*pTaskType = TMQ_DELAYED_TASK__HB;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
......@@ -307,6 +306,7 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
*pTaskType = TMQ_DELAYED_TASK__COMMIT;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
void tmqAssignDelayedReportTask(void* param, void* tmrId) {
......@@ -314,6 +314,7 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) {
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
*pTaskType = TMQ_DELAYED_TASK__REPORT;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
......@@ -364,7 +365,6 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
pParam->rspErr = code;
tmq_t* tmq = pParam->tmq;
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__SUBSCRIBED);
tsem_post(&pParam->rspSem);
return 0;
}
......@@ -385,14 +385,16 @@ tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
}
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i);
tmq_list_append(*topics, strdup(topic->topicName));
tmq_list_append(*topics, topic->topicName);
}
return TMQ_RESP_ERR__SUCCESS;
}
tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) {
tmq_list_t* lst = tmq_list_new();
return tmq_subscribe(tmq, lst);
tmq_list_t* lst = tmq_list_new();
tmq_resp_err_t rsp = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst);
return rsp;
}
#if 0
......@@ -475,7 +477,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
strcpy(pTmq->groupId, conf->groupId);
pTmq->autoCommit = conf->autoCommit;
pTmq->autoCommitInterval = conf->autoCommitInterval;
pTmq->commit_cb = conf->commit_cb;
pTmq->commit_cb = conf->commitCb;
pTmq->resetOffsetCfg = conf->resetOffset;
// assign consumerId
......@@ -655,6 +657,9 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
// avoid double free if msg is sent
buf = NULL;
tsem_wait(&param.rspSem);
tsem_destroy(&param.rspSem);
......@@ -686,7 +691,7 @@ FAIL:
void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
//
conf->commit_cb = cb;
conf->commitCb = cb;
}
TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
......@@ -798,7 +803,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
// do not write into queue since updating epoch reset
tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch,
tmqEpoch);
/*tsem_post(&tmq->rspSem);*/
tsem_post(&tmq->rspSem);
return 0;
}
......@@ -806,25 +811,6 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
}
#if 0
if (pParam->sync == 1) {
/**pParam->msg = taosMemoryMalloc(sizeof(tmq_message_t));*/
*pParam->msg = taosAllocateQitem(sizeof(tmq_message_t));
if (*pParam->msg) {
memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp));
if ((*pParam->msg)->consumeRsp.numOfTopics != 0) {
pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset;
}
taosWriteQitem(tmq->mqueue, *pParam->msg);
tsem_post(&pParam->rspSem);
return 0;
}
tsem_post(&pParam->rspSem);
return -1;
}
#endif
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper));
if (pRspWrapper == NULL) {
tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
......@@ -843,14 +829,14 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset);
taosWriteQitem(tmq->mqueue, pRspWrapper);
/*tsem_post(&tmq->rspSem);*/
tsem_post(&tmq->rspSem);
return 0;
CREATE_MSG_FAIL:
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
/*tsem_post(&tmq->rspSem);*/
tsem_post(&tmq->rspSem);
return -1;
}
......@@ -927,6 +913,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics);
taosHashCleanup(pHash);
tmq->clientTopics = newTopics;
if (taosArrayGetSize(tmq->clientTopics) == 0)
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC);
else
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
atomic_store_32(&tmq->epoch, epoch);
return set;
}
......@@ -955,9 +947,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
/*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
/*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
if (tmqUpdateEp(tmq, head->epoch, &rsp)) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
}
tmqUpdateEp(tmq, head->epoch, &rsp);
tDeleteSMqAskEpRsp(&rsp);
} else {
SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper));
......@@ -972,7 +962,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg);
taosWriteQitem(tmq->mqueue, pWrapper);
/*tsem_post(&tmq->rspSem);*/
tsem_post(&tmq->rspSem);
taosMemoryFree(pParam);
}
......@@ -1076,7 +1066,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
return TMQ_RESP_ERR__FAIL;
}
SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t waitTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
int64_t reqOffset;
if (pVg->currentOffset >= 0) {
reqOffset = pVg->currentOffset;
......@@ -1088,7 +1078,7 @@ SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClient
reqOffset = tmq->resetOffsetCfg;
}
SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2));
SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq));
if (pReq == NULL) {
return NULL;
}
......@@ -1101,14 +1091,14 @@ SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClient
pReq->subKey[tlen] = TMQ_SEPARATOR;
strcpy(pReq->subKey + tlen + 1, pTopic->topicName);
pReq->blockingTime = blockingTime;
pReq->waitTime = waitTime;
pReq->consumerId = tmq->consumerId;
pReq->epoch = tmq->epoch;
pReq->currentOffset = reqOffset;
pReq->reqId = generateRequestId();
pReq->head.vgId = htonl(pVg->vgId);
pReq->head.contLen = htonl(sizeof(SMqPollReqV2));
pReq->head.contLen = htonl(sizeof(SMqPollReq));
return pReq;
}
......@@ -1130,7 +1120,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
return pRspObj;
}
int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
int32_t tmqPollImpl(tmq_t* tmq, int64_t waitTime) {
/*printf("call poll\n");*/
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
......@@ -1151,17 +1141,17 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
#endif
}
atomic_store_32(&pVg->vgSkipCnt, 0);
SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, waitTime, pTopic, pVg);
if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
/*tsem_post(&tmq->rspSem);*/
tsem_post(&tmq->rspSem);
return -1;
}
SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
if (pParam == NULL) {
taosMemoryFree(pReq);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
/*tsem_post(&tmq->rspSem);*/
tsem_post(&tmq->rspSem);
return -1;
}
pParam->tmq = tmq;
......@@ -1169,20 +1159,19 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
pParam->pTopic = pTopic;
pParam->vgId = pVg->vgId;
pParam->epoch = tmq->epoch;
pParam->sync = 0;
SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
taosMemoryFree(pReq);
taosMemoryFree(pParam);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
/*tsem_post(&tmq->rspSem);*/
tsem_post(&tmq->rspSem);
return -1;
}
sendInfo->msgInfo = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqPollReqV2),
.len = sizeof(SMqPollReq),
.handle = NULL,
};
sendInfo->requestId = pReq->reqId;
......@@ -1222,7 +1211,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset)
return 0;
}
SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) {
while (1) {
SMqRspWrapper* rspWrapper = NULL;
taosGetQitem(tmq->qall, (void**)&rspWrapper);
......@@ -1261,37 +1250,41 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
taosFreeQitem(rspWrapper);
if (pollIfReset && reset) {
tscDebug("consumer %ld reset and repoll", tmq->consumerId);
tmqPollImpl(tmq, blockingTime);
tmqPollImpl(tmq, waitTime);
}
}
}
}
TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
SMqRspObj* rspObj;
int64_t startTime = taosGetTimestampMs();
rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
rspObj = tmqHandleAllRsp(tmq, wait_time, false);
if (rspObj) {
return (TAOS_RES*)rspObj;
}
if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) {
return NULL;
}
while (1) {
tmqHandleAllDelayedTask(tmq);
tmqPollImpl(tmq, blocking_time);
/*tsem_wait(&tmq->rspSem);*/
tmqPollImpl(tmq, wait_time);
rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
rspObj = tmqHandleAllRsp(tmq, wait_time, false);
if (rspObj) {
return (TAOS_RES*)rspObj;
}
if (blocking_time != 0) {
if (wait_time != 0) {
int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) {
int64_t leftTime = endTime - startTime;
if (leftTime > wait_time) {
tscDebug("consumer %ld (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch);
return NULL;
}
tsem_timewait(&tmq->rspSem, leftTime * 1000);
}
}
}
......
source/common/src/tmsg.c
浏览文件 @ 68f7c0ec
......@@ -517,7 +517,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1;
if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1;
if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1;
......@@ -527,6 +526,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
}
for (int32_t i = 0; i < pReq->numOfTags; ++i) {
......@@ -534,13 +534,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
}
for (int32_t i = 0; i < pReq->numOfSmas; ++i) {
SField *pField = taosArrayGet(pReq->pSmas, i);
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
}
if (pReq->commentLen > 0) {
......@@ -571,15 +565,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1;
pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
pReq->pSmas = taosArrayInit(pReq->numOfSmas, sizeof(SField));
if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pSmas == NULL) {
if (pReq->pColumns == NULL || pReq->pTags == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
......@@ -589,6 +581,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
if (taosArrayPush(pReq->pColumns, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
......@@ -600,23 +593,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
if (taosArrayPush(pReq->pTags, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
for (int32_t i = 0; i < pReq->numOfSmas; ++i) {
SField field = {0};
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (taosArrayPush(pReq->pSmas, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
if (pReq->commentLen > 0) {
pReq->comment = taosMemoryMalloc(pReq->commentLen);
if (pReq->comment == NULL) return -1;
......@@ -644,13 +627,11 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
void tFreeSMCreateStbReq(SMCreateStbReq *pReq) {
taosArrayDestroy(pReq->pColumns);
taosArrayDestroy(pReq->pTags);
taosArrayDestroy(pReq->pSmas);
taosMemoryFreeClear(pReq->comment);
taosMemoryFreeClear(pReq->pAst1);
taosMemoryFreeClear(pReq->pAst2);
pReq->pColumns = NULL;
pReq->pTags = NULL;
pReq->pSmas = NULL;
}
int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
......
source/dnode/mgmt/exe/dmMain.c
浏览文件 @ 68f7c0ec
......@@ -69,8 +69,8 @@ static void dmSetSignalHandle() {
static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
int32_t cmdEnvIndex = 0;
if (argc < 2) return 0;
global.envCmd = taosMemoryMalloc(argc-1);
memset(global.envCmd, 0, argc-1);
global.envCmd = taosMemoryMalloc((argc-1)*sizeof(char*));
memset(global.envCmd, 0, (argc-1)*sizeof(char*));
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
......
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
浏览文件 @ 68f7c0ec
......@@ -114,7 +114,6 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->tsdbCfg.keep2 = 3650; // pCreate->daysToKeep0;
pCfg->tsdbCfg.keep0 = 3650; // pCreate->daysToKeep2;
pCfg->tsdbCfg.keep1 = 3650; // pCreate->daysToKeep0;
pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
pCfg->tsdbCfg.retentions = pCreate->pRetensions;
pCfg->walCfg.vgId = pCreate->vgId;
pCfg->hashBegin = pCreate->hashBegin;
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ 68f7c0ec
......@@ -126,8 +126,6 @@ typedef enum {
DND_REASON_OTHERS
} EDndReason;
typedef void (*TransCbFp)(SMnode* pMnode, void* param);
typedef struct {
int32_t id;
ETrnStage stage;
......@@ -150,8 +148,10 @@ typedef struct {
int64_t dbUid;
char dbname[TSDB_DB_FNAME_LEN];
char lastError[TSDB_TRANS_ERROR_LEN];
TransCbFp transCbFp;
void* transCbParam;
int32_t startFunc;
int32_t stopFunc;
int32_t paramLen;
void* param;
} STrans;
typedef struct {
......@@ -358,13 +358,11 @@ typedef struct {
int32_t ttl;
int32_t numOfColumns;
int32_t numOfTags;
int32_t numOfSmas;
int32_t commentLen;
int32_t ast1Len;
int32_t ast2Len;
SSchema* pColumns;
SSchema* pTags;
SSchema* pSmas;
char* comment;
char* pAst1;
char* pAst2;
......
source/dnode/mnode/impl/inc/mndInt.h
浏览文件 @ 68f7c0ec
......@@ -40,12 +40,12 @@ extern "C" {
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg);
typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode);
typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter);
typedef struct SQWorkerMgmt SQHandle;
......@@ -84,32 +84,32 @@ typedef struct {
int64_t timeseriesAllowed;
} SGrantInfo;
struct SMnode {
int32_t selfId;
int64_t clusterId;
int8_t replica;
int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA];
tmr_h timer;
tmr_h transTimer;
tmr_h mqTimer;
tmr_h telemTimer;
char *path;
int64_t checkTime;
SSdb *pSdb;
SMgmtWrapper *pWrapper;
SArray *pSteps;
SQHandle *pQuery;
SShowMgmt showMgmt;
SProfileMgmt profileMgmt;
STelemMgmt telemMgmt;
SSyncMgmt syncMgmt;
SHashObj *infosMeta;
SHashObj *perfsMeta;
SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX];
SMsgCb msgCb;
};
typedef struct SMnode {
int32_t selfId;
int64_t clusterId;
int8_t replica;
int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA];
tmr_h timer;
tmr_h transTimer;
tmr_h mqTimer;
tmr_h telemTimer;
char *path;
int64_t checkTime;
SSdb *pSdb;
SMgmtWrapper *pWrapper;
SArray *pSteps;
SQHandle *pQuery;
SShowMgmt showMgmt;
SProfileMgmt profileMgmt;
STelemMgmt telemMgmt;
SSyncMgmt syncMgmt;
SHashObj *infosMeta;
SHashObj *perfsMeta;
SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX];
SMsgCb msgCb;
} SMnode;
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
int64_t mndGenerateUid(char *name, int32_t len);
......
source/dnode/mnode/impl/inc/mndTrans.h
浏览文件 @ 68f7c0ec
......@@ -33,6 +33,15 @@ typedef struct {
void *pCont;
} STransAction;
typedef enum {
TEST_TRANS_START_FUNC = 1,
TEST_TRANS_STOP_FUNC = 2,
CONSUME_TRANS_START_FUNC = 3,
CONSUME_TRANS_STOP_FUNC = 4,
} ETrnFuncType;
typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);
int32_t mndInitTrans(SMnode *pMnode);
void mndCleanupTrans(SMnode *pMnode);
......@@ -44,7 +53,7 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction);
int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param);
void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen);
void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb);
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
......
source/dnode/mnode/impl/src/mndStb.c
浏览文件 @ 68f7c0ec
......@@ -72,8 +72,8 @@ void mndCleanupStb(SMnode *pMnode) {}
SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) +
+pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen +
pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER;
......@@ -91,7 +91,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER)
......@@ -112,14 +111,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
}
for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
SSchema *pSchema = &pStb->pSmas[i];
SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER)
SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER)
SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
}
if (pStb->commentLen > 0) {
SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER)
}
......@@ -178,15 +169,13 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER)
pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema));
pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema));
pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema));
if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) {
if (pStb->pColumns == NULL || pStb->pTags == NULL) {
goto _OVER;
}
......@@ -206,14 +195,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
}
for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
SSchema *pSchema = &pStb->pSmas[i];
SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER)
SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
}
if (pStb->commentLen > 0) {
pStb->comment = taosMemoryCalloc(pStb->commentLen, 1);
if (pStb->comment == NULL) goto _OVER;
......@@ -291,18 +272,6 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
}
}
if (pOld->numOfSmas < pNew->numOfSmas) {
void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema));
if (pSmas != NULL) {
taosMemoryFree(pOld->pSmas);
pOld->pSmas = pSmas;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
taosWUnLockLatch(&pOld->lock);
}
}
if (pOld->commentLen < pNew->commentLen) {
void *comment = taosMemoryMalloc(pNew->commentLen);
if (comment != NULL) {
......@@ -411,11 +380,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
req.schemaTag.nCols = pStb->numOfTags;
req.schemaTag.pSchema = pStb->pTags;
// TODO: remove here
for (int iCol = 0; iCol < req.schema.nCols; iCol++) {
req.schema.pSchema[iCol].flags = SCHEMA_SMA_ON;
}
if (req.rollup) {
req.pRSmaParam.xFilesFactor = pStb->xFilesFactor;
req.pRSmaParam.delay = pStb->delay;
......@@ -674,7 +638,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
stbObj.ttl = pCreate->ttl;
stbObj.numOfColumns = pCreate->numOfColumns;
stbObj.numOfTags = pCreate->numOfTags;
stbObj.numOfSmas = pCreate->numOfSmas;
stbObj.commentLen = pCreate->commentLen;
if (stbObj.commentLen > 0) {
stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1);
......@@ -707,8 +670,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema));
stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema));
stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema));
if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) {
if (stbObj.pColumns == NULL || stbObj.pTags == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
......@@ -718,6 +680,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
SSchema *pSchema = &stbObj.pColumns[i];
pSchema->type = pField->type;
pSchema->bytes = pField->bytes;
pSchema->flags = pField->flags;
memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
pSchema->colId = stbObj.nextColId;
stbObj.nextColId++;
......@@ -733,18 +696,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
stbObj.nextColId++;
}
for (int32_t i = 0; i < stbObj.numOfSmas; ++i) {
SField *pField = taosArrayGet(pCreate->pSmas, i);
SSchema *pSchema = &stbObj.pSmas[i];
SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name);
if (pColSchema == NULL) {
mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name);
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}
memcpy(pSchema, pColSchema, sizeof(SSchema));
}
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg);
if (pTrans == NULL) goto _OVER;
......
source/dnode/mnode/impl/src/mndSync.c
浏览文件 @ 68f7c0ec
......@@ -22,13 +22,15 @@ static int32_t mndInitWal(SMnode *pMnode) {
char path[PATH_MAX] = {0};
snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
SWalCfg cfg = {.vgId = 1,
.fsyncPeriod = 0,
.rollPeriod = -1,
.segSize = -1,
.retentionPeriod = -1,
.retentionSize = -1,
.level = TAOS_WAL_FSYNC};
SWalCfg cfg = {
.vgId = 1,
.fsyncPeriod = 0,
.rollPeriod = -1,
.segSize = -1,
.retentionPeriod = -1,
.retentionSize = -1,
.level = TAOS_WAL_FSYNC,
};
pMgmt->pWal = walOpen(path, &cfg);
if (pMgmt->pWal == NULL) return -1;
......@@ -54,62 +56,62 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
int64_t first = walGetFirstVer(pWal);
int64_t last = walGetLastVer(pWal);
mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last);
mDebug("start to restore wal, sdbver:%" PRId64 ", first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last);
first = TMAX(lastSdbVer + 1, first);
for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) {
if (walReadWithHandle(pHandle, ver) < 0) {
mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver);
goto WAL_RESTORE_OVER;
mError("ver:%" PRId64 ", failed to read from wal since %s", ver, terrstr());
goto _OVER;
}
SWalHead *pHead = pHandle->pHead;
int64_t sdbVer = sdbUpdateVer(pSdb, 0);
if (sdbVer + 1 != ver) {
terrno = TSDB_CODE_SDB_INVALID_WAl_VER;
mError("failed to read wal from sdb, sdbVer:%" PRId64 " inconsistent with ver:%" PRId64, sdbVer, ver);
goto WAL_RESTORE_OVER;
mError("ver:%" PRId64 ", failed to write to sdb, since inconsistent with sdbver:%" PRId64, ver, sdbVer);
goto _OVER;
}
mTrace("wal:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body);
mTrace("ver:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body);
if (sdbWriteWithoutFree(pSdb, (void *)pHead->head.body) < 0) {
mError("failed to read wal from sdb since %s, ver:%" PRId64, terrstr(), ver);
goto WAL_RESTORE_OVER;
mError("ver:%" PRId64 ", failed to write to sdb since %s", ver, terrstr());
goto _OVER;
}
sdbUpdateVer(pSdb, 1);
mDebug("wal:%" PRId64 ", is restored", ver);
mDebug("ver:%" PRId64 ", is restored", ver);
}
int64_t sdbVer = sdbUpdateVer(pSdb, 0);
mDebug("restore sdb wal finished, sdb ver:%" PRId64, sdbVer);
mDebug("restore wal finished, sdbver:%" PRId64, sdbVer);
mndTransPullup(pMnode);
sdbVer = sdbUpdateVer(pSdb, 0);
mDebug("pullup trans finished, sdb ver:%" PRId64, sdbVer);
mDebug("pullup trans finished, sdbver:%" PRId64, sdbVer);
if (sdbVer != lastSdbVer) {
mInfo("sdb restored from %" PRId64 " to %" PRId64 ", write file", lastSdbVer, sdbVer);
if (sdbWriteFile(pSdb) != 0) {
goto WAL_RESTORE_OVER;
goto _OVER;
}
if (walCommit(pWal, sdbVer) != 0) {
goto WAL_RESTORE_OVER;
goto _OVER;
}
if (walBeginSnapshot(pWal, sdbVer) < 0) {
goto WAL_RESTORE_OVER;
goto _OVER;
}
if (walEndSnapshot(pWal) < 0) {
goto WAL_RESTORE_OVER;
goto _OVER;
}
}
code = 0;
WAL_RESTORE_OVER:
_OVER:
walCloseReadHandle(pHandle);
return code;
}
......@@ -158,11 +160,11 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) {
int64_t ver = sdbUpdateVer(pSdb, 1);
if (walWrite(pWal, ver, 1, pRaw, sdbGetRawTotalSize(pRaw)) < 0) {
sdbUpdateVer(pSdb, -1);
mError("failed to write raw:%p since %s, ver:%" PRId64, pRaw, terrstr(), ver);
mError("ver:%" PRId64 ", failed to write raw:%p to wal since %s", ver, pRaw, terrstr());
return -1;
}
mTrace("raw:%p, write to wal, ver:%" PRId64, pRaw, ver);
mTrace("ver:%" PRId64 ", write to wal, raw:%p", ver, pRaw);
walCommit(pWal, ver);
walFsync(pWal, true);
......
source/dnode/mnode/impl/src/mndTelem.c
浏览文件 @ 68f7c0ec
......@@ -146,7 +146,6 @@ int32_t mndInitTelem(SMnode* pMnode) {
taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
mDebug("mnode telemetry is initialized");
return 0;
}
......
source/dnode/mnode/impl/src/mndTrans.c
浏览文件 @ 68f7c0ec
......@@ -29,7 +29,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans);
static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld);
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans);
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc);
static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction);
......@@ -174,6 +174,13 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER)
}
SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER)
if (pTrans->param != NULL) {
SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER)
}
SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER)
SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER)
......@@ -305,6 +312,14 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
action.pCont = NULL;
}
SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER)
if (pTrans->paramLen != 0) {
pTrans->param = taosMemoryMalloc(pTrans->paramLen);
SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER);
}
SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER)
terrno = 0;
......@@ -413,9 +428,36 @@ static const char *mndTransType(ETrnType type) {
}
}
static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) {
mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen);
}
static void mndTransTestStopFunc(SMnode *pMnode, void *param, int32_t paramLen) {
mInfo("test trans stop, param:%s, len:%d", (char *)param, paramLen);
}
static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) {
switch (ftype) {
case TEST_TRANS_START_FUNC:
return mndTransTestStartFunc;
case TEST_TRANS_STOP_FUNC:
return mndTransTestStopFunc;
default:
return NULL;
}
}
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
// pTrans->stage = TRN_STAGE_PREPARE;
mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage));
if (pTrans->startFunc > 0) {
TransCbFp fp = mndTransGetCbFp(pTrans->startFunc);
if (fp) {
(*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
}
}
return 0;
}
......@@ -430,10 +472,23 @@ static void mndTransDropData(STrans *pTrans) {
pTrans->rpcRsp = NULL;
pTrans->rpcRspLen = 0;
}
if (pTrans->param != NULL) {
taosMemoryFree(pTrans->param);
pTrans->param = NULL;
pTrans->paramLen = 0;
}
}
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
mTrace("trans:%d, perform delete action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage));
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) {
mDebug("trans:%d, perform delete action, row:%p stage:%s callfunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage),
callFunc);
if (pTrans->stopFunc > 0 && callFunc) {
TransCbFp fp = mndTransGetCbFp(pTrans->stopFunc);
if (fp) {
(*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
}
}
mndTransDropData(pTrans);
return 0;
}
......@@ -498,7 +553,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S
return NULL;
}
mDebug("trans:%d, is created, data:%p", pTrans->id, pTrans);
mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans);
return pTrans;
}
......@@ -525,7 +580,7 @@ static void mndTransDropActions(SArray *pArray) {
void mndTransDrop(STrans *pTrans) {
if (pTrans != NULL) {
mndTransDropData(pTrans);
mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans);
taosMemoryFreeClear(pTrans);
}
}
......@@ -574,9 +629,11 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen) {
pTrans->rpcRspLen = contLen;
}
void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param) {
pTrans->transCbFp = fp;
pTrans->transCbParam = param;
void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen) {
pTrans->startFunc = startFunc;
pTrans->stopFunc = stopFunc;
pTrans->param = param;
pTrans->paramLen = paramLen;
}
void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) {
......@@ -712,8 +769,6 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
pNew->rpcRefId = pTrans->rpcRefId;
pNew->rpcRsp = pTrans->rpcRsp;
pNew->rpcRspLen = pTrans->rpcRspLen;
pNew->transCbFp = pTrans->transCbFp;
pNew->transCbParam = pTrans->transCbParam;
pTrans->rpcRsp = NULL;
pTrans->rpcRspLen = 0;
......@@ -1125,10 +1180,6 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes);
if (pTrans->transCbFp != NULL) {
(*pTrans->transCbFp)(pMnode, pTrans->transCbParam);
}
return continueExec;
}
......
source/dnode/mnode/impl/src/mndUser.c
浏览文件 @ 68f7c0ec
......@@ -21,8 +21,8 @@
#include "mndTrans.h"
#include "tbase64.h"
#define TSDB_USER_VER_NUMBER 1
#define TSDB_USER_RESERVE_SIZE 64
#define USER_VER_NUMBER 1
#define USER_RESERVE_SIZE 64
static int32_t mndCreateDefaultUsers(SMnode *pMnode);
static SSdbRaw *mndUserActionEncode(SUserObj *pUser);
......@@ -35,7 +35,7 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq);
static int32_t mndProcessAlterUserReq(SNodeMsg *pReq);
static int32_t mndProcessDropUserReq(SNodeMsg *pReq);
static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq);
static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextUser(SMnode *pMnode, void *pIter);
int32_t mndInitUser(SMnode *pMnode) {
......@@ -93,39 +93,39 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
int32_t size = sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, size);
if (pRaw == NULL) goto USER_ENCODE_OVER;
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER;
int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_ENCODE_OVER)
SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_ENCODE_OVER)
SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_ENCODE_OVER)
SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, USER_ENCODE_OVER)
SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, USER_ENCODE_OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->superUser, USER_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, USER_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, USER_ENCODE_OVER)
SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER)
SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER)
SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER)
char *db = taosHashIterate(pUser->readDbs, NULL);
while (db != NULL) {
SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER);
db = taosHashIterate(pUser->readDbs, db);
}
db = taosHashIterate(pUser->writeDbs, NULL);
while (db != NULL) {
SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER);
db = taosHashIterate(pUser->writeDbs, db);
}
SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_ENCODE_OVER)
SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER)
SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
terrno = 0;
USER_ENCODE_OVER:
_OVER:
if (terrno != 0) {
mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, terrstr());
sdbFreeRaw(pRaw);
......@@ -140,55 +140,54 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != TSDB_USER_VER_NUMBER) {
if (sver != USER_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto USER_DECODE_OVER;
goto _OVER;
}
SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj));
if (pRow == NULL) goto USER_DECODE_OVER;
if (pRow == NULL) goto _OVER;
SUserObj *pUser = sdbGetRowObj(pRow);
if (pUser == NULL) goto USER_DECODE_OVER;
pUser->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
pUser->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto USER_DECODE_OVER;
if (pUser == NULL) goto _OVER;
int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_DECODE_OVER)
SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_DECODE_OVER)
SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_DECODE_OVER)
SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, USER_DECODE_OVER)
SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, USER_DECODE_OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, USER_DECODE_OVER)
SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER)
int32_t numOfReadDbs = 0;
int32_t numOfWriteDbs = 0;
SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, USER_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, USER_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER)
pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
pUser->writeDbs = taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER;
for (int32_t i = 0; i < numOfReadDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0};
SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER)
SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER)
int32_t len = strlen(db) + 1;
taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN);
}
for (int32_t i = 0; i < numOfWriteDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0};
SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER)
SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER)
int32_t len = strlen(db) + 1;
taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN);
}
SDB_GET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_DECODE_OVER)
SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
terrno = 0;
USER_DECODE_OVER:
_OVER:
if (terrno != 0) {
mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr());
taosHashCleanup(pUser->readDbs);
......@@ -220,6 +219,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser);
taosHashCleanup(pUser->readDbs);
taosHashCleanup(pUser->writeDbs);
pUser->readDbs = NULL;
pUser->writeDbs = NULL;
return 0;
}
......@@ -228,13 +229,8 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
pOld->updateTime = pNew->updateTime;
void *tmp1 = pOld->readDbs;
pOld->readDbs = pNew->readDbs;
pNew->readDbs = tmp1;
void *tmp2 = pOld->writeDbs;
pOld->writeDbs = pNew->writeDbs;
pNew->writeDbs = tmp2;
TSWAP(pOld->readDbs, pNew->readDbs, (void *));
TSWAP(pOld->writeDbs, pNew->writeDbs, (void *));
return 0;
}
......@@ -277,6 +273,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
char *param = strdup("====> test code to be deleted later <=====");
mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
......@@ -296,41 +295,41 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) {
if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto CREATE_USER_OVER;
goto _OVER;
}
mDebug("user:%s, start to create", createReq.user);
if (createReq.user[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
goto CREATE_USER_OVER;
goto _OVER;
}
if (createReq.pass[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
goto CREATE_USER_OVER;
goto _OVER;
}
pUser = mndAcquireUser(pMnode, createReq.user);
if (pUser != NULL) {
terrno = TSDB_CODE_MND_USER_ALREADY_EXIST;
goto CREATE_USER_OVER;
goto _OVER;
}
pOperUser = mndAcquireUser(pMnode, pReq->user);
if (pOperUser == NULL) {
terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
goto CREATE_USER_OVER;
goto _OVER;
}
if (mndCheckCreateUserAuth(pOperUser) != 0) {
goto CREATE_USER_OVER;
goto _OVER;
}
code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq);
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
CREATE_USER_OVER:
_OVER:
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to create since %s", createReq.user, terrstr());
}
......@@ -399,38 +398,38 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto ALTER_USER_OVER;
goto _OVER;
}
mDebug("user:%s, start to alter", alterReq.user);
if (alterReq.user[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
goto ALTER_USER_OVER;
goto _OVER;
}
if (alterReq.pass[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
goto ALTER_USER_OVER;
goto _OVER;
}
pUser = mndAcquireUser(pMnode, alterReq.user);
if (pUser == NULL) {
terrno = TSDB_CODE_MND_USER_NOT_EXIST;
goto ALTER_USER_OVER;
goto _OVER;
}
pOperUser = mndAcquireUser(pMnode, pReq->user);
if (pOperUser == NULL) {
terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
goto ALTER_USER_OVER;
goto _OVER;
}
memcpy(&newUser, pUser, sizeof(SUserObj));
newUser.readDbs = mndDupDbHash(pUser->readDbs);
newUser.writeDbs = mndDupDbHash(pUser->writeDbs);
if (newUser.readDbs == NULL || newUser.writeDbs == NULL) {
goto ALTER_USER_OVER;
goto _OVER;
}
int32_t len = strlen(alterReq.dbname) + 1;
......@@ -446,50 +445,50 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
} else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB) {
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto ALTER_USER_OVER;
goto _OVER;
}
if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto ALTER_USER_OVER;
goto _OVER;
}
} else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) {
if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto ALTER_USER_OVER;
goto _OVER;
}
} else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) {
taosHashClear(newUser.readDbs);
} else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) {
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto ALTER_USER_OVER;
goto _OVER;
}
if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto ALTER_USER_OVER;
goto _OVER;
}
} else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) {
if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto ALTER_USER_OVER;
goto _OVER;
}
} else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) {
taosHashClear(newUser.writeDbs);
} else {
terrno = TSDB_CODE_MND_INVALID_ALTER_OPER;
goto ALTER_USER_OVER;
goto _OVER;
}
newUser.updateTime = taosGetTimestampMs();
if (mndCheckAlterUserAuth(pOperUser, pUser, pDb, &alterReq) != 0) {
goto ALTER_USER_OVER;
goto _OVER;
}
code = mndUpdateUser(pMnode, pUser, &newUser, pReq);
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
ALTER_USER_OVER:
_OVER:
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to alter since %s", alterReq.user, terrstr());
}
......@@ -537,36 +536,36 @@ static int32_t mndProcessDropUserReq(SNodeMsg *pReq) {
if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto DROP_USER_OVER;
goto _OVER;
}
mDebug("user:%s, start to drop", dropReq.user);
if (dropReq.user[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
goto DROP_USER_OVER;
goto _OVER;
}
pUser = mndAcquireUser(pMnode, dropReq.user);
if (pUser == NULL) {
terrno = TSDB_CODE_MND_USER_NOT_EXIST;
goto DROP_USER_OVER;
goto _OVER;
}
pOperUser = mndAcquireUser(pMnode, pReq->user);
if (pOperUser == NULL) {
terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
goto DROP_USER_OVER;
goto _OVER;
}
if (mndCheckDropUserAuth(pOperUser) != 0) {
goto DROP_USER_OVER;
goto _OVER;
}
code = mndDropUser(pMnode, pReq, pUser);
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
DROP_USER_OVER:
_OVER:
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to drop since %s", dropReq.user, terrstr());
}
......@@ -586,7 +585,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto GET_AUTH_OVER;
goto _OVER;
}
mTrace("user:%s, start to get auth", authReq.user);
......@@ -594,7 +593,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
pUser = mndAcquireUser(pMnode, authReq.user);
if (pUser == NULL) {
terrno = TSDB_CODE_MND_USER_NOT_EXIST;
goto GET_AUTH_OVER;
goto _OVER;
}
memcpy(authRsp.user, pUser->user, TSDB_USER_LEN);
......@@ -622,7 +621,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
void *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto GET_AUTH_OVER;
goto _OVER;
}
tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp);
......@@ -631,7 +630,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
pReq->rspLen = contLen;
code = 0;
GET_AUTH_OVER:
_OVER:
mndReleaseUser(pMnode, pUser);
taosHashCleanup(authRsp.readDbs);
taosHashCleanup(authRsp.writeDbs);
......@@ -639,7 +638,7 @@ GET_AUTH_OVER:
return code;
}
static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
......@@ -652,29 +651,29 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pB
if (pShow->pIter == NULL) break;
cols = 0;
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->bytes[cols]);
colDataAppend(pColInfo, numOfRows, (const char*) name, false);
colDataAppend(pColInfo, numOfRows, (const char *)name, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
const char* src = pUser->superUser? "super":"normal";
char b[10+VARSTR_HEADER_SIZE] = {0};
const char *src = pUser->superUser ? "super" : "normal";
char b[10 + VARSTR_HEADER_SIZE] = {0};
STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src));
colDataAppend(pColInfo, numOfRows, (const char*) b, false);
colDataAppend(pColInfo, numOfRows, (const char *)b, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
colDataAppend(pColInfo, numOfRows, (const char*) &pUser->createdTime, false);
colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->acct, pShow->bytes[cols]);
colDataAppend(pColInfo, numOfRows, (const char*) name, false);
colDataAppend(pColInfo, numOfRows, (const char *)name, false);
numOfRows++;
sdbRelease(pSdb, pUser);
......
source/dnode/mnode/impl/src/mndVgroup.c
浏览文件 @ 68f7c0ec
......@@ -45,8 +45,8 @@ int32_t mndInitVgroup(SMnode *pMnode) {
.encodeFp = (SdbEncodeFp)mndVgroupActionEncode,
.decodeFp = (SdbDecodeFp)mndVgroupActionDecode,
.insertFp = (SdbInsertFp)mndVgroupActionInsert,
.updateFp = (SdbUpdateFp)mndVgroupActionDelete,
.deleteFp = (SdbDeleteFp)mndVgroupActionUpdate};
.updateFp = (SdbUpdateFp)mndVgroupActionUpdate,
.deleteFp = (SdbDeleteFp)mndVgroupActionDelete};
mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp);
mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp);
......
source/dnode/mnode/impl/test/sma/sma.cpp
浏览文件 @ 68f7c0ec
......@@ -114,18 +114,15 @@ void* MndTestSma::BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen)
SMCreateStbReq createReq = {0};
createReq.numOfColumns = 3;
createReq.numOfTags = 1;
createReq.numOfSmas = 1;
createReq.igExists = 0;
createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField));
createReq.pSmas = taosArrayInit(createReq.numOfSmas, sizeof(SField));
strcpy(createReq.name, stbname);
PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts");
PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1");
PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2");
PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1");
PushField(createReq.pSmas, 2, TSDB_DATA_TYPE_TINYINT, "col1");
int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
void* pHead = rpcMallocCont(tlen);
......@@ -190,7 +187,7 @@ void* MndTestSma::BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int3
}
TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) {
#if 0
#if 0
const char* dbname = "1.d1";
const char* stbname = "1.d1.stb";
const char* smaname = "1.d1.sma";
......@@ -244,7 +241,7 @@ TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) {
test.SendShowRetrieveReq();
EXPECT_EQ(test.GetShowRows(), 0);
}
#endif
#endif
}
TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
......@@ -264,7 +261,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildCreateBSmaStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1);
}
......@@ -280,7 +277,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildDropStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 0);
}
......
source/dnode/mnode/sdb/inc/sdbInt.h
浏览文件 @ 68f7c0ec
......@@ -19,10 +19,6 @@
#include "os.h"
#include "sdb.h"
#include "thash.h"
#include "tlockfree.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
......@@ -53,26 +49,6 @@ typedef struct SSdbRow {
char pObj[];
} SSdbRow;
typedef struct SSdb {
SMnode *pMnode;
char *currDir;
char *syncDir;
char *tmpDir;
int64_t lastCommitVer;
int64_t curVer;
int64_t tableVer[SDB_MAX];
int64_t maxId[SDB_MAX];
EKeyType keyTypes[SDB_MAX];
SHashObj *hashObjs[SDB_MAX];
SRWLatch locks[SDB_MAX];
SdbInsertFp insertFps[SDB_MAX];
SdbUpdateFp updateFps[SDB_MAX];
SdbDeleteFp deleteFps[SDB_MAX];
SdbDeployFp deployFps[SDB_MAX];
SdbEncodeFp encodeFps[SDB_MAX];
SdbDecodeFp decodeFps[SDB_MAX];
} SSdb;
const char *sdbTableName(ESdbType type);
void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);
......
source/dnode/mnode/sdb/src/sdb.c
浏览文件 @ 68f7c0ec
......@@ -87,7 +87,7 @@ void sdbCleanup(SSdb *pSdb) {
SSdbRow *pRow = *ppRow;
if (pRow == NULL) continue;
sdbFreeRow(pSdb, pRow);
sdbFreeRow(pSdb, pRow, true);
ppRow = taosHashIterate(hash, ppRow);
}
}
......@@ -162,7 +162,4 @@ static int32_t sdbCreateDir(SSdb *pSdb) {
return 0;
}
int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) {
pSdb->curVer += val;
return pSdb->curVer;
}
\ No newline at end of file
int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { return atomic_add_fetch_64(&pSdb->curVer, val); }
\ No newline at end of file
source/dnode/mnode/sdb/src/sdbFile.c
浏览文件 @ 68f7c0ec
......@@ -28,7 +28,7 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
if (fp == NULL) continue;
if ((*fp)(pSdb->pMnode) != 0) {
mError("failed to deploy sdb:%d since %s", i, terrstr());
mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr());
return -1;
}
}
......
source/dnode/mnode/sdb/src/sdbHash.c
浏览文件 @ 68f7c0ec
......@@ -137,7 +137,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize);
if (pOldRow != NULL) {
taosWUnLockLatch(pLock);
sdbFreeRow(pSdb, pRow);
sdbFreeRow(pSdb, pRow, false);
terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
return terrno;
}
......@@ -148,7 +148,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) {
taosWUnLockLatch(pLock);
sdbFreeRow(pSdb, pRow);
sdbFreeRow(pSdb, pRow, false);
terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
return terrno;
}
......@@ -164,7 +164,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosWLockLatch(pLock);
taosHashRemove(hash, pRow->pObj, keySize);
taosWUnLockLatch(pLock);
sdbFreeRow(pSdb, pRow);
sdbFreeRow(pSdb, pRow, false);
terrno = code;
return terrno;
}
......@@ -202,7 +202,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj);
}
sdbFreeRow(pSdb, pNewRow);
sdbFreeRow(pSdb, pNewRow, false);
pSdb->tableVer[pOldRow->type]++;
return code;
......@@ -215,7 +215,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize);
if (ppOldRow == NULL || *ppOldRow == NULL) {
taosWUnLockLatch(pLock);
sdbFreeRow(pSdb, pRow);
sdbFreeRow(pSdb, pRow, false);
terrno = TSDB_CODE_SDB_OBJ_NOT_THERE;
return terrno;
}
......@@ -228,7 +228,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosWUnLockLatch(pLock);
pSdb->tableVer[pOldRow->type]++;
sdbFreeRow(pSdb, pRow);
sdbFreeRow(pSdb, pRow, false);
sdbCheck(pSdb, pOldRow);
// sdbRelease(pSdb, pOldRow->pObj);
......@@ -322,7 +322,7 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) {
int32_t ref = atomic_load_32(&pRow->refCount);
sdbPrintOper(pSdb, pRow, "check");
if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
sdbFreeRow(pSdb, pRow);
sdbFreeRow(pSdb, pRow, true);
}
taosRUnLockLatch(pLock);
......@@ -340,7 +340,7 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "release");
if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
sdbFreeRow(pSdb, pRow);
sdbFreeRow(pSdb, pRow, true);
}
taosRUnLockLatch(pLock);
......
source/dnode/mnode/sdb/src/sdbRaw.c
浏览文件 @ 68f7c0ec
......@@ -107,7 +107,9 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_
return -1;
}
memcpy(pRaw->pData + dataPos, pVal, valLen);
if (pVal != NULL) {
memcpy(pRaw->pData + dataPos, pVal, valLen);
}
return 0;
}
......
source/dnode/mnode/sdb/src/sdbRow.c
浏览文件 @ 68f7c0ec
......@@ -36,11 +36,11 @@ void *sdbGetRowObj(SSdbRow *pRow) {
return pRow->pObj;
}
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) {
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc) {
// remove attached object such as trans
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
if (deleteFp != NULL) {
(*deleteFp)(pSdb, pRow->pObj);
(*deleteFp)(pSdb, pRow->pObj, callFunc);
}
sdbPrintOper(pSdb, pRow, "free");
......
source/dnode/vnode/CMakeLists.txt
浏览文件 @ 68f7c0ec
......@@ -19,7 +19,6 @@ target_sources(
"src/meta/metaOpen.c"
"src/meta/metaIdx.c"
"src/meta/metaTable.c"
"src/meta/metaTDBImpl.c"
"src/meta/metaQuery.c"
"src/meta/metaCommit.c"
"src/meta/metaEntry.c"
......@@ -28,12 +27,10 @@ target_sources(
"src/tsdb/tsdbTDBImpl.c"
"src/tsdb/tsdbCommit.c"
"src/tsdb/tsdbCommit2.c"
"src/tsdb/tsdbCompact.c"
"src/tsdb/tsdbFile.c"
"src/tsdb/tsdbFS.c"
"src/tsdb/tsdbMain.c"
"src/tsdb/tsdbOpen.c"
"src/tsdb/tsdbMemTable.c"
"src/tsdb/tsdbOptions.c"
"src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c"
"src/tsdb/tsdbScan.c"
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ 68f7c0ec
......@@ -75,8 +75,9 @@ typedef struct SMeta SMeta; // todo: remove
typedef struct SMetaReader SMetaReader;
typedef struct SMetaEntry SMetaEntry;
void metaReaderInit(SMetaReader *pReader, SVnode *pVnode, int32_t flags);
void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
void metaReaderClear(SMetaReader *pReader);
int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
int metaReadNext(SMetaReader *pReader);
#if 1 // refact APIs below (TODO)
......@@ -91,8 +92,8 @@ int metaTbCursorNext(SMTbCursor *pTbCur);
#endif
// tsdb
typedef struct STsdb STsdb;
typedef void *tsdbReaderT;
typedef struct STsdb STsdb;
typedef void *tsdbReaderT;
#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
......@@ -112,7 +113,7 @@ bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis);
SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond);
void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond);
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
......@@ -134,23 +135,18 @@ int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *p
// need to reposition
// structs
struct SMetaCfg {
uint64_t lruSize;
};
struct STsdbCfg {
int8_t precision;
int8_t update;
int8_t compression;
int8_t slLevel;
int32_t days;
int32_t minRows;
int32_t maxRows;
int32_t keep2;
int32_t keep0;
int32_t keep1;
uint64_t lruCacheSize;
SArray *retentions;
int8_t precision;
int8_t update;
int8_t compression;
int8_t slLevel;
int32_t days;
int32_t minRows;
int32_t maxRows;
int32_t keep0;
int32_t keep1;
int32_t keep2;
SArray *retentions;
};
struct SVnodeCfg {
......@@ -161,8 +157,6 @@ struct SVnodeCfg {
int32_t szCache;
uint64_t szBuf;
bool isHeap;
uint32_t ttl;
uint32_t keep;
int8_t streamMode;
bool isWeak;
STsdbCfg tsdbCfg;
......
source/dnode/vnode/src/inc/meta.h
浏览文件 @ 68f7c0ec
......@@ -16,13 +16,14 @@
#ifndef _TD_VNODE_META_H_
#define _TD_VNODE_META_H_
#include "vnodeInt.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SMetaIdx SMetaIdx;
typedef struct SMetaDB SMetaDB;
typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMSmaCursor SMSmaCursor;
// metaDebug ==================
......@@ -36,22 +37,16 @@ typedef struct SMSmaCursor SMSmaCursor;
// clang-format on
// metaOpen ==================
int metaOpen(SVnode* pVnode, SMeta** ppMeta);
int metaClose(SMeta* pMeta);
// metaEntry ==================
int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME);
int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME);
// metaTable ==================
int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
// metaQuery ==================
int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid);
int metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid);
int metaGetTableEntryByName(SMetaReader* pReader, const char* name);
// metaIdx ==================
int metaOpenIdx(SMeta* pMeta);
......@@ -60,8 +55,6 @@ int metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
// metaCommit ==================
int metaBegin(SMeta* pMeta);
static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
struct SMeta {
......@@ -106,27 +99,12 @@ typedef struct {
} STtlIdxKey;
#if 1
#define META_SUPER_TABLE TD_SUPER_TABLE
#define META_CHILD_TABLE TD_CHILD_TABLE
#define META_NORMAL_TABLE TD_NORMAL_TABLE
// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaDropTable(SMeta* pMeta, tb_uid_t uid);
int metaCommit(SMeta* pMeta);
int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup);
int metaGetTbNum(SMeta* pMeta);
SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseSmaCursor(SMSmaCursor* pSmaCur);
int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur);
SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
int metaDropTable(SMeta* pMeta, tb_uid_t uid);
SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseSmaCursor(SMSmaCursor* pSmaCur);
int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur);
#ifndef META_REFACT
// SMetaDB
......
source/dnode/vnode/src/inc/tq.h
浏览文件 @ 68f7c0ec
......@@ -16,6 +16,8 @@
#ifndef _TD_VNODE_TQ_H_
#define _TD_VNODE_TQ_H_
#include "vnodeInt.h"
#include "executor.h"
#include "os.h"
#include "tcache.h"
......@@ -31,12 +33,12 @@ extern "C" {
// tqDebug ===================
// clang-format off
#define tqFatal(...) do { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
#define tqError(...) do { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
#define tqWarn(...) do { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
#define tqInfo(...) do { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
#define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqFatal(...) do { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
#define tqError(...) do { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
#define tqWarn(...) do { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
#define tqInfo(...) do { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
#define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
#define TQ_BUFFER_SIZE 4
......@@ -237,17 +239,7 @@ int tqInit();
void tqCleanUp();
// open in each vnode
STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
void tqClose(STQ*);
// required by vnode
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqCommit(STQ*);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**);
int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**);
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ 68f7c0ec
......@@ -16,6 +16,8 @@
#ifndef _TD_VNODE_TSDB_H_
#define _TD_VNODE_TSDB_H_
#include "vnodeInt.h"
#ifdef __cplusplus
extern "C" {
#endif
......@@ -43,14 +45,46 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKe
TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
// tsdbCommit ================
int tsdbBegin(STsdb *pTsdb);
#if 1
// tsdbFS ================
typedef struct STsdbFS STsdbFS;
typedef struct SSmaStat SSmaStat;
// tsdbSma ================
typedef struct SSmaEnv SSmaEnv;
typedef struct SSmaEnvs SSmaEnvs;
// structs
typedef struct {
int minFid;
int midFid;
int maxFid;
TSKEY minKey;
} SRtn;
struct SSmaEnvs {
int16_t nTSma;
int16_t nRSma;
SSmaEnv *pTSmaEnv;
SSmaEnv *pRSmaEnv;
};
struct STsdb {
char *path;
SVnode *pVnode;
bool repoLocked;
TdThreadMutex mutex;
STsdbCfg config;
STsdbMemTable *mem;
STsdbMemTable *imem;
SRtn rtn;
STsdbFS *fs;
SSmaEnvs smaEnvs;
};
#if 1 // ======================================
typedef struct SSmaStat SSmaStat;
struct STable {
uint64_t tid;
uint64_t uid;
......@@ -60,16 +94,9 @@ struct STable {
#define TABLE_TID(t) (t)->tid
#define TABLE_UID(t) (t)->uid
int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb);
int tsdbClose(STsdb *pTsdb);
int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp);
int tsdbPrepareCommit(STsdb *pTsdb);
int tsdbCommit(STsdb *pTsdb);
int32_t tsdbInitSma(STsdb *pTsdb);
int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg);
int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version);
int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg);
int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
......@@ -103,13 +130,6 @@ typedef struct {
uint8_t state;
} SDFile;
struct SSmaEnvs {
int16_t nTSma;
int16_t nRSma;
SSmaEnv *pTSmaEnv;
SSmaEnv *pRSmaEnv;
};
typedef struct {
int fid;
int8_t state; // -128~127
......@@ -118,13 +138,6 @@ typedef struct {
SDFile files[TSDB_FILE_MAX];
} SDFileSet;
typedef struct {
int minFid;
int midFid;
int maxFid;
TSKEY minKey;
} SRtn;
struct STbData {
tb_uid_t uid;
TSKEY keyMin;
......@@ -161,7 +174,7 @@ typedef struct {
SArray *sf; // sma data file array v2f1900.index_name_1
} SFSStatus;
typedef struct {
struct STsdbFS {
TdThreadRwlock lock;
SFSStatus *cstatus; // current status
......@@ -169,23 +182,9 @@ typedef struct {
SHashObj *metaCacheComp; // meta cache for compact
bool intxn;
SFSStatus *nstatus; // new status
} STsdbFS;
struct STsdb {
char *path;
SVnode *pVnode;
int32_t vgId;
bool repoLocked;
TdThreadMutex mutex;
STsdbCfg config;
STsdbMemTable *mem;
STsdbMemTable *imem;
SRtn rtn;
STsdbFS *fs;
SSmaEnvs smaEnvs;
};
#define REPO_ID(r) ((r)->vgId)
#define REPO_ID(r) TD_VID((r)->pVnode)
#define REPO_CFG(r) (&(r)->config)
#define REPO_FS(r) ((r)->fs)
#define REPO_META(r) ((r)->pVnode->pMeta)
......@@ -237,12 +236,6 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
return TD_ROW_KEY(row);
}
// tsdbOptions
extern const STsdbCfg defautlTsdbOptions;
int tsdbValidateOptions(const STsdbCfg *);
void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc);
// tsdbReadImpl
typedef struct SReadH SReadH;
......@@ -519,12 +512,6 @@ static FORCE_INLINE void *taosTZfree(void *ptr) {
// tsdbCommit
typedef struct {
uint64_t uid;
int64_t offset;
int64_t size;
} SKVRecord;
void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
......@@ -903,66 +890,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
return 0;
}
// tsdbSma
// #define TSDB_SMA_TEST // remove after test finished
// struct SSmaEnv {
// TdThreadRwlock lock;
// SDiskID did;
// TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level?
// char *path; // relative path
// SSmaStat *pStat;
// };
// #define SMA_ENV_LOCK(env) ((env)->lock)
// #define SMA_ENV_DID(env) ((env)->did)
// #define SMA_ENV_ENV(env) ((env)->dbEnv)
// #define SMA_ENV_PATH(env) ((env)->path)
// #define SMA_ENV_STAT(env) ((env)->pStat)
// #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
// void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
// void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
// #if 0
// int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
// int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
// #endif
// // internal func
// static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
// int32_t len = 0;
// len += taosEncodeFixedI64(pData, tsKey);
// len += taosEncodeFixedI64(pData, groupId);
// return len;
// }
// static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
// int code = taosThreadRwlockRdlock(&(pEnv->lock));
// if (code != 0) {
// terrno = TAOS_SYSTEM_ERROR(code);
// return -1;
// }
// return 0;
// }
// static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
// int code = taosThreadRwlockWrlock(&(pEnv->lock));
// if (code != 0) {
// terrno = TAOS_SYSTEM_ERROR(code);
// return -1;
// }
// return 0;
// }
// static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
// int code = taosThreadRwlockUnlock(&(pEnv->lock));
// if (code != 0) {
// terrno = TAOS_SYSTEM_ERROR(code);
// return -1;
// }
// return 0;
// }
typedef struct SSmaKey SSmaKey;
struct SSmaKey {
......
source/dnode/vnode/src/inc/tsdbSma.h
浏览文件 @ 68f7c0ec
......@@ -16,9 +16,7 @@
#ifndef _TD_VNODE_TSDB_SMA_H_
#define _TD_VNODE_TSDB_SMA_H_
#include "os.h"
#include "thash.h"
#include "tmsg.h"
#include "tsdb.h"
#ifdef __cplusplus
extern "C" {
......@@ -32,12 +30,6 @@ struct STbDdlH {
__tb_ddl_fn_t fp;
};
typedef struct {
tb_uid_t suid;
SArray *tbUids;
SHashObj *uidHash;
} STbUidStore;
static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
ASSERT(*pStore == NULL);
*pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
......
source/dnode/vnode/src/inc/vnd.h
浏览文件 @ 68f7c0ec
......@@ -16,6 +16,10 @@
#ifndef _TD_VND_H_
#define _TD_VND_H_
#include "sync.h"
#include "syncTools.h"
#include "vnodeInt.h"
#ifdef __cplusplus
extern "C" {
#endif
......@@ -58,11 +62,9 @@ struct SVBufPool {
SVBufPoolNode node;
};
int vnodeOpenBufPool(SVnode* pVnode, int64_t size);
int vnodeCloseBufPool(SVnode* pVnode);
void vnodeBufPoolReset(SVBufPool* pPool);
void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
void vnodeBufPoolFree(SVBufPool* pPool, void* p);
int vnodeOpenBufPool(SVnode* pVnode, int64_t size);
int vnodeCloseBufPool(SVnode* pVnode);
void vnodeBufPoolReset(SVBufPool* pPool);
// vnodeQuery ====================
int vnodeQueryOpen(SVnode* pVnode);
......@@ -79,6 +81,20 @@ int vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo);
int vnodeSyncCommit(SVnode* pVnode);
int vnodeAsyncCommit(SVnode* pVnode);
// vnodeCommit ====================
int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
int32_t vnodeSyncStart(SVnode* pVnode);
void vnodeSyncClose(SVnode* pVnode);
void vnodeSyncSetQ(SVnode* pVnode, void* qHandle);
void vnodeSyncSetRpc(SVnode* pVnode, void* rpcHandle);
int32_t vnodeSyncEqMsg(void* qHandle, SRpcMsg* pMsg);
int32_t vnodeSendMsg(void* rpcHandle, const SEpSet* pEpSet, SRpcMsg* pMsg);
void vnodeSyncCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
void vnodeSyncPreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
void vnodeSyncRollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
SSyncFSM* syncVnodeMakeFsm();
#ifdef __cplusplus
}
#endif
......
source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @ 68f7c0ec
......@@ -59,6 +59,54 @@ typedef struct SQWorkerMgmt SQHandle;
#define VNODE_TQ_DIR "tq"
#define VNODE_WAL_DIR "wal"
// vnd.h
void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
void vnodeBufPoolFree(SVBufPool* pPool, void* p);
// meta
typedef struct SMCtbCursor SMCtbCursor;
typedef struct STbUidStore STbUidStore;
int metaOpen(SVnode* pVnode, SMeta** ppMeta);
int metaClose(SMeta* pMeta);
int metaBegin(SMeta* pMeta);
int metaCommit(SMeta* pMeta);
int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
int metaGetTableEntryByName(SMetaReader* pReader, const char* name);
int metaGetTbNum(SMeta* pMeta);
SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup);
void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
// tsdb
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb);
int tsdbClose(STsdb* pTsdb);
int tsdbBegin(STsdb* pTsdb);
int tsdbCommit(STsdb* pTsdb);
int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
// tq
STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqCommit(STQ*);
int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
typedef struct {
int8_t streamType; // sma or other
int8_t dstType;
......@@ -106,6 +154,12 @@ struct SVnode {
SQHandle* pQuery;
};
struct STbUidStore {
tb_uid_t suid;
SArray* tbUids;
SHashObj* uidHash;
};
#define TD_VID(PVNODE) (PVNODE)->config.vgId
typedef struct STbDdlH STbDdlH;
......@@ -113,18 +167,6 @@ typedef struct STbDdlH STbDdlH;
// sma
void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
#include "vnd.h"
#include "meta.h"
#include "tsdb.h"
#include "tq.h"
#include "vnodeSync.h"
#include "tsdbSma.h"
#ifdef __cplusplus
}
#endif
......
source/dnode/vnode/src/inc/vnodeSync.h 已删除 100644 → 0
浏览文件 @ d62ddcf2
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_SYNC_H_
#define _TD_VNODE_SYNC_H_
#ifdef __cplusplus
extern "C" {
#endif
int32_t vnodeSyncOpen(SVnode *pVnode, char *path);
int32_t vnodeSyncStart(SVnode *pVnode);
void vnodeSyncClose(SVnode *pVnode);
void vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg);
int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg);
void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot);
SSyncFSM *syncVnodeMakeFsm();
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_SYNC_H_*/
source/dnode/vnode/src/meta/metaCommit.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "meta.h"
static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBufPoolMalloc((SVBufPool *)pPool, size); }
static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }
......
source/dnode/vnode/src/meta/metaEntry.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "meta.h"
int metaEncodeEntry(SCoder *pCoder, const SMetaEntry *pME) {
if (tStartEncode(pCoder) < 0) return -1;
......
source/dnode/vnode/src/meta/metaIdx.c
浏览文件 @ 68f7c0ec
......@@ -16,7 +16,7 @@
#ifdef USE_INVERTED_INDEX
#include "index.h"
#endif
#include "vnodeInt.h"
#include "meta.h"
struct SMetaIdx {
#ifdef USE_INVERTED_INDEX
......
source/dnode/vnode/src/meta/metaOpen.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "meta.h"
static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
......
source/dnode/vnode/src/meta/metaQuery.c
浏览文件 @ 68f7c0ec
......@@ -13,12 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "meta.h"
void metaReaderInit(SMetaReader *pReader, SVnode *pVnode, int32_t flags) {
void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) {
memset(pReader, 0, sizeof(*pReader));
pReader->flags = flags;
pReader->pMeta = pVnode->pMeta;
pReader->pMeta = pMeta;
}
void metaReaderClear(SMetaReader *pReader) {
......@@ -32,6 +32,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u
// query table.db
if (tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
goto _err;
}
......@@ -54,6 +55,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
// query uid.idx
if (tdbDbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
}
......@@ -67,6 +69,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
// query name.idx
if (tdbDbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
}
......@@ -91,7 +94,7 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
return NULL;
}
metaReaderInit(&pTbCur->mr, pMeta->pVnode, 0);
metaReaderInit(&pTbCur->mr, pMeta, 0);
tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc);
......@@ -122,7 +125,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
}
metaGetTableEntryByVersion(&pTbCur->mr, *(int64_t *)pTbCur->pVal, *(tb_uid_t *)pTbCur->pKey);
if (pTbCur->mr.me.type == META_SUPER_TABLE) {
if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) {
continue;
}
......@@ -234,7 +237,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
STSchemaBuilder sb = {0};
SSchema *pSchema;
metaReaderInit(&mr, pMeta->pVnode, 0);
metaReaderInit(&mr, pMeta, 0);
metaGetTableEntryByUid(&mr, uid);
if (mr.me.type == TSDB_CHILD_TABLE) {
......
source/dnode/vnode/src/meta/metaTable.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "meta.h"
static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
......@@ -37,7 +37,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaReader mr = {0};
// validate req
metaReaderInit(&mr, pMeta->pVnode, 0);
metaReaderInit(&mr, pMeta, 0);
if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
// TODO: just for pass case
#if 0
......@@ -91,7 +91,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
pReq->ctime = taosGetTimestampMs();
// validate req
metaReaderInit(&mr, pMeta->pVnode, 0);
metaReaderInit(&mr, pMeta, 0);
if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
metaReaderClear(&mr);
......
source/dnode/vnode/src/tq/tq.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "tq.h"
int32_t tqInit() {
//
......@@ -78,7 +78,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
SMqDataBlkRsp rsp = {0};
rsp.reqOffset = pExec->pushHandle.reqOffset;
rsp.blockData = taosArrayInit(0, sizeof(int32_t));
rsp.blockData = taosArrayInit(0, sizeof(void*));
rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
......@@ -176,9 +176,9 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
atomic_store_ptr(&pExec->pushHandle.handle, NULL);
taosWUnLockLatch(&pExec->pushHandle.lock);
vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum,
rsp.reqOffset, rsp.rspOffset);
tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum,
rsp.reqOffset, rsp.rspOffset);
// TODO destroy
taosArrayDestroy(rsp.blockData);
......@@ -210,35 +210,6 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req);
#if 0
void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL);
while (pIter != NULL) {
STqPusher* pusher = *(STqPusher**)pIter;
if (pusher->type == TQ_PUSHER_TYPE__STREAM) {
STqStreamPusher* streamPusher = (STqStreamPusher*)pusher;
// repack
STqStreamToken* token = taosMemoryMalloc(sizeof(STqStreamToken));
if (token == NULL) {
taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
token->type = TQ_STREAM_TOKEN__DATA;
token->data = msg;
// set input
// exec
}
// send msg to ep
}
// iterate hash
// process all msg
// if waiting
// memcpy and send msg to fetch thread
// TODO: add reference
// if handle waiting, launch query and response to consumer
//
// if no waiting handle, return
#endif
return 0;
}
......@@ -375,11 +346,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
}
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
SMqPollReqV2* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId;
int64_t waitTime = pReq->blockingTime;
int32_t reqEpoch = pReq->epoch;
int64_t fetchOffset;
SMqPollReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId;
int64_t waitTime = pReq->waitTime;
int32_t reqEpoch = pReq->epoch;
int64_t fetchOffset;
// get offset to fetch message
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
......@@ -390,8 +361,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
fetchOffset = pReq->currentOffset + 1;
}
vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey));
ASSERT(pExec);
......@@ -418,16 +389,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
while (1) {
consumerEpoch = atomic_load_32(&pExec->epoch);
if (consumerEpoch > reqEpoch) {
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
break;
}
taosThreadMutexLock(&pExec->pWalReader->mutex);
if (walFetchHead(pExec->pWalReader, fetchOffset, pHeadWithCkSum) < 0) {
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset);
tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset);
taosThreadMutexUnlock(&pExec->pWalReader->mutex);
break;
}
......@@ -448,7 +419,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
// TODO: no more log, set timer to wait blocking time
// if data inserted during waiting, launch query and
// response to user
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset);
#if 0
......@@ -476,8 +447,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
}
#endif
vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
if (pHead->msgType == TDMT_VND_SUBMIT) {
SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
......@@ -591,8 +562,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
pMsg->code = 0;
tmsgSendRsp(pMsg);
vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
// TODO destroy
taosArrayDestroy(rsp.blockData);
......@@ -618,7 +589,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
fetchOffset = pReq->currentOffset + 1;
}
vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
SMqPollRspV2 rspV2 = {0};
......@@ -660,7 +631,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
return 0;
}
vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
tqDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
TD_VID(pTq->pVnode));
rspV2.reqOffset = pReq->currentOffset;
......@@ -671,7 +642,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
// TODO
consumerEpoch = atomic_load_32(&pConsumer->epoch);
if (consumerEpoch > reqEpoch) {
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
break;
}
......@@ -680,11 +651,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
// TODO: no more log, set timer to wait blocking time
// if data inserted during waiting, launch query and
// response to user
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset);
break;
}
vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
/*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
/*pHead = pTopic->pReadhandle->pHead;*/
......@@ -709,7 +680,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
}
if (taosArrayGetSize(pRes) == 0) {
vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
fetchOffset++;
rspV2.skipLogNum++;
......@@ -770,7 +741,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
pMsg->pCont = buf;
pMsg->contLen = msgLen;
pMsg->code = 0;
vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
pHead->msgType, consumerId, pReq->epoch);
tmsgSendRsp(pMsg);
taosMemoryFree(pHead);
......@@ -804,7 +775,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
pMsg->contLen = tlen;
pMsg->code = 0;
tmsgSendRsp(pMsg);
vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
pReq->epoch);
/*}*/
......
source/dnode/vnode/src/tq/tqCommit.c
浏览文件 @ 68f7c0ec
......@@ -13,4 +13,4 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "tq.h"
source/dnode/vnode/src/tq/tqMetaStore.c
浏览文件 @ 68f7c0ec
......@@ -12,7 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "tq.h"
// #include <fcntl.h>
// #include <string.h>
// #include <unistd.h>
......@@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
}
strcpy(pMeta->dirPath, path);
char *name = taosMemoryMalloc(pathLen + 10) ;
char* name = taosMemoryMalloc(pathLen + 10);
strcpy(name, path);
if (!taosDirExist(name) && taosMkDir(name) != 0) {
......
source/dnode/vnode/src/tq/tqOffset.c
浏览文件 @ 68f7c0ec
......@@ -14,7 +14,7 @@
*/
#define _DEFAULT_SOURCE
#include "vnodeInt.h"
#include "tq.h"
enum ETqOffsetPersist {
TQ_OFFSET_PERSIST__LAZY = 1,
......
source/dnode/vnode/src/tq/tqRead.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "tq.h"
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle));
......
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "tsdb.h"
#define TSDB_MAX_SUBBLOCKS 8
......@@ -752,334 +752,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
return 0;
}
// // =================== Commit Meta Data
// static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) {
// STsdbFS * pfs = REPO_FS(pRepo);
// SMFile * pOMFile = pfs->cstatus->pmf;
// SDiskID did;
// // Create/Open a meta file or open the existing file
// if (pOMFile == NULL) {
// // Create a new meta file
// did.level = TFS_PRIMARY_LEVEL;
// did.id = TFS_PRIMARY_ID;
// tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
// if (open && tsdbCreateMFile(pMf, true) < 0) {
// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
// return -1;
// }
// tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf));
// } else {
// tsdbInitMFileEx(pMf, pOMFile);
// if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) {
// tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno));
// return -1;
// }
// }
// return 0;
// }
// static int tsdbCommitMeta(STsdbRepo *pRepo) {
// STsdbFS * pfs = REPO_FS(pRepo);
// SMemTable *pMem = pRepo->imem;
// SMFile * pOMFile = pfs->cstatus->pmf;
// SMFile mf;
// SActObj * pAct = NULL;
// SActCont * pCont = NULL;
// SListNode *pNode = NULL;
// ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0);
// if (listNEles(pMem->actList) <= 0) {
// // no meta data to commit, just keep the old meta file
// tsdbUpdateMFile(pfs, pOMFile);
// if (tsTsdbMetaCompactRatio > 0) {
// if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) {
// return -1;
// }
// int ret = tsdbCompactMetaFile(pRepo, pfs, &mf);
// if (ret < 0) tsdbError("compact meta file error");
// return ret;
// }
// return 0;
// } else {
// if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) {
// return -1;
// }
// }
// // Loop to write
// while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
// pAct = (SActObj *)pNode->data;
// if (pAct->act == TSDB_UPDATE_META) {
// pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
// if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) {
// tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
// tstrerror(terrno));
// tsdbCloseMFile(&mf);
// (void)tsdbApplyMFileChange(&mf, pOMFile);
// // TODO: need to reload metaCache
// return -1;
// }
// } else if (pAct->act == TSDB_DROP_META) {
// if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) {
// tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
// tstrerror(terrno));
// tsdbCloseMFile(&mf);
// tsdbApplyMFileChange(&mf, pOMFile);
// // TODO: need to reload metaCache
// return -1;
// }
// } else {
// ASSERT(false);
// }
// }
// if (tsdbUpdateMFileHeader(&mf) < 0) {
// tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
// tsdbApplyMFileChange(&mf, pOMFile);
// // TODO: need to reload metaCache
// return -1;
// }
// TSDB_FILE_FSYNC(&mf);
// tsdbCloseMFile(&mf);
// tsdbUpdateMFile(pfs, &mf);
// if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) {
// tsdbError("compact meta file error");
// }
// return 0;
// }
// int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) {
// int tlen = 0;
// tlen += taosEncodeFixedU64(buf, pRecord->uid);
// tlen += taosEncodeFixedI64(buf, pRecord->offset);
// tlen += taosEncodeFixedI64(buf, pRecord->size);
// return tlen;
// }
// void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) {
// buf = taosDecodeFixedU64(buf, &(pRecord->uid));
// buf = taosDecodeFixedI64(buf, &(pRecord->offset));
// buf = taosDecodeFixedI64(buf, &(pRecord->size));
// return buf;
// }
// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) {
// char buf[64] = "\0";
// void * pBuf = buf;
// SKVRecord rInfo;
// int64_t offset;
// // Seek to end of meta file
// offset = tsdbSeekMFile(pMFile, 0, SEEK_END);
// if (offset < 0) {
// return -1;
// }
// rInfo.offset = offset;
// rInfo.uid = uid;
// rInfo.size = contLen;
// int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo);
// if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) {
// return -1;
// }
// if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) {
// return -1;
// }
// tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)));
// SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache;
// pMFile->info.nRecords++;
// SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid));
// if (pRecord != NULL) {
// pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord));
// } else {
// pMFile->info.nRecords++;
// }
// taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo));
// return 0;
// }
// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
// SKVRecord rInfo = {0};
// char buf[128] = "\0";
// SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid));
// if (pRecord == NULL) {
// tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid);
// return -1;
// }
// rInfo.offset = -pRecord->offset;
// rInfo.uid = pRecord->uid;
// rInfo.size = pRecord->size;
// void *pBuf = buf;
// tsdbEncodeKVRecord(&pBuf, &rInfo);
// if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) {
// return -1;
// }
// pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord));
// pMFile->info.nDels++;
// pMFile->info.nRecords--;
// pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
// taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid));
// return 0;
// }
// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) {
// float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords);
// float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size);
// float compactRatio = (float)(tsTsdbMetaCompactRatio)/100;
// if (delPercent < compactRatio && tombPercent < compactRatio) {
// return 0;
// }
// if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
// tsdbError("open meta file %s compact fail", pMFile->f.rname);
// return -1;
// }
// tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64
// ",size:%" PRId64,
// tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size);
// SMFile mf;
// SDiskID did;
// // first create tmp meta file
// did.level = TFS_PRIMARY_LEVEL;
// did.id = TFS_PRIMARY_ID;
// tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1);
// if (tsdbCreateMFile(&mf, true) < 0) {
// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
// return -1;
// }
// tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
// // second iterator metaCache
// int code = -1;
// int64_t maxBufSize = 1024;
// SKVRecord *pRecord;
// void *pBuf = NULL;
// pBuf = taosMemoryMalloc((size_t)maxBufSize);
// if (pBuf == NULL) {
// goto _err;
// }
// // init Comp
// assert(pfs->metaCacheComp == NULL);
// pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
// if (pfs->metaCacheComp == NULL) {
// goto _err;
// }
// pRecord = taosHashIterate(pfs->metaCache, NULL);
// while (pRecord) {
// if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
// tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
// tstrerror(terrno));
// goto _err;
// }
// if (pRecord->size > maxBufSize) {
// maxBufSize = pRecord->size;
// void* tmp = taosMemoryRealloc(pBuf, (size_t)maxBufSize);
// if (tmp == NULL) {
// goto _err;
// }
// pBuf = tmp;
// }
// int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
// if (nread < 0) {
// tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
// tstrerror(terrno));
// goto _err;
// }
// if (nread < pRecord->size) {
// tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
// REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
// goto _err;
// }
// if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) {
// tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid,
// tstrerror(terrno));
// goto _err;
// }
// pRecord = taosHashIterate(pfs->metaCache, pRecord);
// }
// code = 0;
// _err:
// if (code == 0) TSDB_FILE_FSYNC(&mf);
// tsdbCloseMFile(&mf);
// tsdbCloseMFile(pMFile);
// if (code == 0) {
// // rename meta.tmp -> meta
// tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf),
// TSDB_FILE_FULL_NAME(pMFile)); taosRename(mf.f.aname,pMFile->f.aname); tstrncpy(mf.f.aname, pMFile->f.aname,
// TSDB_FILENAME_LEN); tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN);
// // update current meta file info
// pfs->nstatus->pmf = NULL;
// tsdbUpdateMFile(pfs, &mf);
// taosHashCleanup(pfs->metaCache);
// pfs->metaCache = pfs->metaCacheComp;
// pfs->metaCacheComp = NULL;
// } else {
// // remove meta.tmp file
// taosRemoveFile(mf.f.aname);
// taosHashCleanup(pfs->metaCacheComp);
// pfs->metaCacheComp = NULL;
// }
// taosMemoryFreeClear(pBuf);
// ASSERT(mf.info.nDels == 0);
// ASSERT(mf.info.tombSize == 0);
// tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64,
// code,mf.info.nRecords,mf.info.size);
// return code;
// }
// // =================== Commit Time-Series Data
// #if 0
// static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
// for (int i = 0; i < nIters; i++) {
// TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
// if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true;
// }
// return false;
// }
// #endif
// =================== Commit Time-Series Data
static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
SCommitIter *pIter = pCommith->iters + tid;
TSKEY nextKey = tsdbNextIterKey(pIter->pIter);
......
source/dnode/vnode/src/tsdb/tsdbCommit2.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "tsdb.h"
int tsdbBegin(STsdb *pTsdb) {
STsdbMemTable *pMem;
......
source/dnode/vnode/src/tsdb/tsdbCompact.c 已删除 100644 → 0
浏览文件 @ d62ddcf2
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#if 0
#include "tsdbint.h"
typedef struct {
STable * pTable;
SBlockIdx * pBlkIdx;
SBlockIdx bindex;
SBlockInfo *pInfo;
} STableCompactH;
typedef struct {
SRtn rtn;
SFSIter fsIter;
SArray * tbArray; // table array to cache table obj and block indexes
SReadH readh;
SDFileSet wSet;
SArray * aBlkIdx;
SArray * aSupBlk;
SDataCols *pDataCols;
} SCompactH;
#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet))
#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh))
#define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD)
#define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA)
#define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST)
#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD)
#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL)
#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh))
#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh))
static int tsdbAsyncCompact(STsdbRepo *pRepo);
static void tsdbStartCompact(STsdbRepo *pRepo);
static void tsdbEndCompact(STsdbRepo *pRepo, int eno);
static int tsdbCompactMeta(STsdbRepo *pRepo);
static int tsdbCompactTSData(STsdbRepo *pRepo);
static int tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet);
static bool tsdbShouldCompact(SCompactH *pComph);
static int tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo);
static void tsdbDestroyCompactH(SCompactH *pComph);
static int tsdbInitCompTbArray(SCompactH *pComph);
static void tsdbDestroyCompTbArray(SCompactH *pComph);
static int tsdbCacheFSetIndex(SCompactH *pComph);
static int tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet);
static void tsdbCompactFSetEnd(SCompactH *pComph);
static int tsdbCompactFSetImpl(SCompactH *pComph);
static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
void **ppCBuf);
enum { TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT};
int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); }
void *tsdbCompactImpl(STsdbRepo *pRepo) {
// Check if there are files in TSDB FS to compact
if (REPO_FS(pRepo)->cstatus->pmf == NULL) {
tsdbInfo("vgId:%d no file to compact in FS", REPO_ID(pRepo));
return NULL;
}
tsdbStartCompact(pRepo);
if (tsdbCompactMeta(pRepo) < 0) {
tsdbError("vgId:%d failed to compact META data since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
if (tsdbCompactTSData(pRepo) < 0) {
tsdbError("vgId:%d failed to compact TS data since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
tsdbEndCompact(pRepo, TSDB_CODE_SUCCESS);
return NULL;
_err:
pRepo->code = terrno;
tsdbEndCompact(pRepo, terrno);
return NULL;
}
static int tsdbAsyncCompact(STsdbRepo *pRepo) {
if (pRepo->compactState != TSDB_NO_COMPACT) {
tsdbInfo("vgId:%d not compact tsdb again ", REPO_ID(pRepo));
return 0;
}
pRepo->compactState = TSDB_WAITING_COMPACT;
tsem_wait(&(pRepo->readyToCommit));
return tsdbScheduleCommit(pRepo, COMPACT_REQ);
}
static void tsdbStartCompact(STsdbRepo *pRepo) {
assert(pRepo->compactState != TSDB_IN_COMPACT);
tsdbInfo("vgId:%d start to compact!", REPO_ID(pRepo));
tsdbStartFSTxn(pRepo, 0, 0);
pRepo->code = TSDB_CODE_SUCCESS;
pRepo->compactState = TSDB_IN_COMPACT;
}
static void tsdbEndCompact(STsdbRepo *pRepo, int eno) {
if (eno != TSDB_CODE_SUCCESS) {
tsdbEndFSTxnWithError(REPO_FS(pRepo));
} else {
tsdbEndFSTxn(pRepo);
}
pRepo->compactState = TSDB_NO_COMPACT;
tsdbInfo("vgId:%d compact over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
tsem_post(&(pRepo->readyToCommit));
}
static int tsdbCompactMeta(STsdbRepo *pRepo) {
STsdbFS *pfs = REPO_FS(pRepo);
tsdbUpdateMFile(pfs, pfs->cstatus->pmf);
return 0;
}
static int tsdbCompactTSData(STsdbRepo *pRepo) {
SCompactH compactH;
SDFileSet *pSet = NULL;
tsdbDebug("vgId:%d start to compact TS data", REPO_ID(pRepo));
// If no file, just return 0;
if (taosArrayGetSize(REPO_FS(pRepo)->cstatus->df) <= 0) {
tsdbDebug("vgId:%d no TS data file to compact, compact over", REPO_ID(pRepo));
return 0;
}
if (tsdbInitCompactH(&compactH, pRepo) < 0) {
return -1;
}
while ((pSet = tsdbFSIterNext(&(compactH.fsIter)))) {
// Remove those expired files
if (pSet->fid < compactH.rtn.minFid) {
tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
continue;
}
if (TSDB_FSET_LEVEL(pSet) == TFS_MAX_LEVEL) {
tsdbDebug("vgId:%d FSET %d on level %d, should not compact", REPO_ID(pRepo), pSet->fid, TFS_MAX_LEVEL);
tsdbUpdateDFileSet(REPO_FS(pRepo), pSet);
continue;
}
if (tsdbCompactFSet(&compactH, pSet) < 0) {
tsdbDestroyCompactH(&compactH);
tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
return -1;
}
}
tsdbDestroyCompactH(&compactH);
tsdbDebug("vgId:%d compact TS data over", REPO_ID(pRepo));
return 0;
}
static int tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet) {
STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
SDiskID did;
tsdbDebug("vgId:%d start to compact FSET %d on level %d id %d", REPO_ID(pRepo), pSet->fid, TSDB_FSET_LEVEL(pSet),
TSDB_FSET_ID(pSet));
if (tsdbCompactFSetInit(pComph, pSet) < 0) {
return -1;
}
if (!tsdbShouldCompact(pComph)) {
tsdbDebug("vgId:%d no need to compact FSET %d", REPO_ID(pRepo), pSet->fid);
if (tsdbApplyRtnOnFSet(TSDB_COMPACT_REPO(pComph), pSet, &(pComph->rtn)) < 0) {
tsdbCompactFSetEnd(pComph);
return -1;
}
} else {
// Create new fset as compacted fset
if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &did) < 0) {
terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
tsdbCompactFSetEnd(pComph);
return -1;
}
tsdbInitDFileSet(TSDB_COMPACT_WSET(pComph), did, REPO_ID(pRepo), TSDB_FSET_FID(pSet),
FS_TXN_VERSION(REPO_FS(pRepo)));
if (tsdbCreateDFileSet(TSDB_COMPACT_WSET(pComph), true) < 0) {
tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
tsdbCompactFSetEnd(pComph);
return -1;
}
if (tsdbCompactFSetImpl(pComph) < 0) {
tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
tsdbRemoveDFileSet(TSDB_COMPACT_WSET(pComph));
tsdbCompactFSetEnd(pComph);
return -1;
}
tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
tsdbUpdateDFileSet(REPO_FS(pRepo), TSDB_COMPACT_WSET(pComph));
tsdbDebug("vgId:%d FSET %d compact over", REPO_ID(pRepo), pSet->fid);
}
tsdbCompactFSetEnd(pComph);
return 0;
}
static bool tsdbShouldCompact(SCompactH *pComph) {
STsdbRepo * pRepo = TSDB_COMPACT_REPO(pComph);
STsdbCfg * pCfg = REPO_CFG(pRepo);
SReadH * pReadh = &(pComph->readh);
STableCompactH *pTh;
SBlock * pBlock;
int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
SDFile * pDataF = TSDB_READ_DATA_FILE(pReadh);
SDFile * pLastF = TSDB_READ_LAST_FILE(pReadh);
int tblocks = 0; // total blocks
int nSubBlocks = 0; // # of blocks with sub-blocks
int nSmallBlocks = 0; // # of blocks with rows < defaultRows
int64_t tsize = 0;
for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
for (size_t bidx = 0; bidx < pTh->pBlkIdx->numOfBlocks; bidx++) {
tblocks++;
pBlock = pTh->pInfo->blocks + bidx;
if (pBlock->numOfRows < defaultRows) {
nSmallBlocks++;
}
if (pBlock->numOfSubBlocks > 1) {
nSubBlocks++;
for (int k = 0; k < pBlock->numOfSubBlocks; k++) {
SBlock *iBlock = ((SBlock *)POINTER_SHIFT(pTh->pInfo, pBlock->offset)) + k;
tsize = tsize + iBlock->len;
}
} else if (pBlock->numOfSubBlocks == 1) {
tsize += pBlock->len;
} else {
ASSERT(0);
}
}
}
return (((nSubBlocks * 1.0 / tblocks) > 0.33) || ((nSmallBlocks * 1.0 / tblocks) > 0.33) ||
(tsize * 1.0 / (pDataF->info.size + pLastF->info.size - 2 * TSDB_FILE_HEAD_SIZE) < 0.85));
}
static int tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo) {
STsdbCfg *pCfg = REPO_CFG(pRepo);
memset(pComph, 0, sizeof(*pComph));
TSDB_FSET_SET_CLOSED(TSDB_COMPACT_WSET(pComph));
tsdbGetRtnSnap(pRepo, &(pComph->rtn));
tsdbFSIterInit(&(pComph->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD);
if (tsdbInitReadH(&(pComph->readh), pRepo) < 0) {
return -1;
}
if (tsdbInitCompTbArray(pComph) < 0) {
tsdbDestroyCompactH(pComph);
return -1;
}
pComph->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
if (pComph->aBlkIdx == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyCompactH(pComph);
return -1;
}
pComph->aSupBlk = taosArrayInit(1024, sizeof(SBlock));
if (pComph->aSupBlk == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyCompactH(pComph);
return -1;
}
pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
if (pComph->pDataCols == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyCompactH(pComph);
return -1;
}
return 0;
}
static void tsdbDestroyCompactH(SCompactH *pComph) {
pComph->pDataCols = tdFreeDataCols(pComph->pDataCols);
pComph->aSupBlk = taosArrayDestroy(pComph->aSupBlk);
pComph->aBlkIdx = taosArrayDestroy(pComph->aBlkIdx);
tsdbDestroyCompTbArray(pComph);
tsdbDestroyReadH(&(pComph->readh));
tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
}
static int tsdbInitCompTbArray(SCompactH *pComph) { // Init pComp->tbArray
STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
STsdbMeta *pMeta = pRepo->tsdbMeta;
if (tsdbRLockRepoMeta(pRepo) < 0) return -1;
pComph->tbArray = taosArrayInit(pMeta->maxTables, sizeof(STableCompactH));
if (pComph->tbArray == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbUnlockRepoMeta(pRepo);
return -1;
}
// Note here must start from 0
for (int i = 0; i < pMeta->maxTables; i++) {
STableCompactH ch = {0};
if (pMeta->tables[i] != NULL) {
tsdbRefTable(pMeta->tables[i]);
ch.pTable = pMeta->tables[i];
}
if (taosArrayPush(pComph->tbArray, &ch) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbUnlockRepoMeta(pRepo);
return -1;
}
}
if (tsdbUnlockRepoMeta(pRepo) < 0) return -1;
return 0;
}
static void tsdbDestroyCompTbArray(SCompactH *pComph) {
STableCompactH *pTh;
if (pComph->tbArray == NULL) return;
for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
if (pTh->pTable) {
tsdbUnRefTable(pTh->pTable);
}
pTh->pInfo = taosTZfree(pTh->pInfo);
}
pComph->tbArray = taosArrayDestroy(pComph->tbArray);
}
static int tsdbCacheFSetIndex(SCompactH *pComph) {
SReadH *pReadH = &(pComph->readh);
if (tsdbLoadBlockIdx(pReadH) < 0) {
return -1;
}
for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
pTh->pBlkIdx = NULL;
if (pTh->pTable == NULL) continue;
if (tsdbSetReadTable(pReadH, pTh->pTable) < 0) {
return -1;
}
if (pReadH->pBlkIdx == NULL) continue;
pTh->bindex = *(pReadH->pBlkIdx);
pTh->pBlkIdx = &(pTh->bindex);
if (tsdbMakeRoom((void **)(&(pTh->pInfo)), pTh->pBlkIdx->len) < 0) {
return -1;
}
if (tsdbLoadBlockInfo(pReadH, (void *)(pTh->pInfo)) < 0) {
return -1;
}
}
return 0;
}
static int tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet) {
taosArrayClear(pComph->aBlkIdx);
taosArrayClear(pComph->aSupBlk);
if (tsdbSetAndOpenReadFSet(&(pComph->readh), pSet) < 0) {
return -1;
}
if (tsdbCacheFSetIndex(pComph) < 0) {
tsdbCloseAndUnsetFSet(&(pComph->readh));
return -1;
}
return 0;
}
static void tsdbCompactFSetEnd(SCompactH *pComph) { tsdbCloseAndUnsetFSet(&(pComph->readh)); }
static int tsdbCompactFSetImpl(SCompactH *pComph) {
STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
STsdbCfg * pCfg = REPO_CFG(pRepo);
SReadH * pReadh = &(pComph->readh);
SBlockIdx blkIdx;
void ** ppBuf = &(TSDB_COMPACT_BUF(pComph));
void ** ppCBuf = &(TSDB_COMPACT_COMP_BUF(pComph));
int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
taosArrayClear(pComph->aBlkIdx);
for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
STSchema * pSchema;
if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
pSchema = tsdbGetTableSchemaImpl(pTh->pTable, true, true, -1);
taosArrayClear(pComph->aSupBlk);
if ((tdInitDataCols(pComph->pDataCols, pSchema) < 0) || (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) ||
(tdInitDataCols(pReadh->pDCols[1], pSchema) < 0)) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
tdFreeSchema(pSchema);
// Loop to compact each block data
for (int i = 0; i < pTh->pBlkIdx->numOfBlocks; i++) {
SBlock *pBlock = pTh->pInfo->blocks + i;
// Load the block data
if (tsdbLoadBlockData(pReadh, pBlock, pTh->pInfo) < 0) {
return -1;
}
// Merge pComph->pDataCols and pReadh->pDCols[0] and write data to file
if (pComph->pDataCols->numOfRows == 0 && pBlock->numOfRows >= defaultRows) {
if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pReadh->pDCols[0], ppBuf, ppCBuf) < 0) {
return -1;
}
} else {
int ridx = 0;
while (true) {
if (pReadh->pDCols[0]->numOfRows - ridx == 0) break;
int rowsToMerge = TMIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows);
tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE);
if (pComph->pDataCols->numOfRows < defaultRows) {
break;
}
if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf) < 0) {
return -1;
}
tdResetDataCols(pComph->pDataCols);
}
}
}
if (pComph->pDataCols->numOfRows > 0 &&
tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf) < 0) {
return -1;
}
if (tsdbWriteBlockInfoImpl(TSDB_COMPACT_HEAD_FILE(pComph), pTh->pTable, pComph->aSupBlk, NULL, ppBuf, &blkIdx) <
0) {
return -1;
}
if ((blkIdx.numOfBlocks > 0) && (taosArrayPush(pComph->aBlkIdx, (void *)(&blkIdx)) == NULL)) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
}
if (tsdbWriteBlockIdx(TSDB_COMPACT_HEAD_FILE(pComph), pComph->aBlkIdx, ppBuf) < 0) {
return -1;
}
return 0;
}
static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
void **ppCBuf) {
STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
STsdbCfg * pCfg = REPO_CFG(pRepo);
SDFile * pDFile;
bool isLast;
SBlock block;
ASSERT(pDataCols->numOfRows > 0);
if (pDataCols->numOfRows < pCfg->minRowsPerFileBlock) {
pDFile = TSDB_COMPACT_LAST_FILE(pComph);
isLast = true;
} else {
pDFile = TSDB_COMPACT_DATA_FILE(pComph);
isLast = false;
}
if (tsdbWriteBlockImpl(pRepo, pTable, pDFile, pDataCols, &block, isLast, true, ppBuf, ppCBuf) < 0) {
return -1;
}
if (taosArrayPush(pComph->aSupBlk, (void *)(&block)) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
return 0;
}
#endif
source/dnode/vnode/src/tsdb/tsdbFS.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "tsdb.h"
typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T;
static const char *tsdbTxnFname[] = {"current.t", "current"};
......@@ -246,62 +246,6 @@ void *tsdbFreeFS(STsdbFS *pfs) {
return NULL;
}
// static int tsdbProcessExpiredFS(STsdb *pRepo) {
// tsdbStartFSTxn(pRepo, 0, 0);
// // if (tsdbCreateMeta(pRepo) < 0) {
// // tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno));
// // return -1;
// // }
// if (tsdbApplyRtn(pRepo) < 0) {
// tsdbEndFSTxnWithError(REPO_FS(pRepo));
// tsdbError("vgId:%d failed to apply rtn since %s", REPO_ID(pRepo), tstrerror(terrno));
// return -1;
// }
// if (tsdbEndFSTxn(pRepo) < 0) {
// tsdbError("vgId:%d failed to end fs txn since %s", REPO_ID(pRepo), tstrerror(terrno));
// return -1;
// }
// return 0;
// }
// static int tsdbCreateMeta(STsdb *pRepo) {
// STsdbFS *pfs = REPO_FS(pRepo);
// SMFile * pOMFile = pfs->cstatus->pmf;
// SMFile mf;
// SDiskID did;
// if (pOMFile != NULL) {
// // keep the old meta file
// tsdbUpdateMFile(pfs, pOMFile);
// return 0;
// }
// // Create a new meta file
// did.level = TFS_PRIMARY_LEVEL;
// did.id = TFS_PRIMARY_ID;
// tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
// if (tsdbCreateMFile(&mf, true) < 0) {
// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
// return -1;
// }
// tsdbInfo("vgId:%d meta file %s is created", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
// if (tsdbUpdateMFileHeader(&mf) < 0) {
// tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
// tsdbApplyMFileChange(&mf, pOMFile);
// return -1;
// }
// TSDB_FILE_FSYNC(&mf);
// tsdbCloseMFile(&mf);
// tsdbUpdateMFile(pfs, &mf);
// return 0;
// }
int tsdbOpenFS(STsdb *pRepo) {
STsdbFS *pfs = REPO_FS(pRepo);
char current[TSDB_FILENAME_LEN] = "\0";
......@@ -644,7 +588,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
}
static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), pRepo->vgId,
snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo),
tsdbTxnFname[ftype]);
}
......@@ -769,142 +713,6 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) {
return 0;
}
// int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta) {
// char tbuf[128];
// STsdbFS * pfs = REPO_FS(pRepo);
// SMFile mf;
// SMFile * pMFile = &mf;
// void * pBuf = NULL;
// SKVRecord rInfo;
// int64_t maxBufSize = 0;
// SMFInfo minfo;
// taosHashClear(pfs->metaCache);
// // No meta file, just return
// if (pfs->cstatus->pmf == NULL) return 0;
// mf = pfs->cstatus->mf;
// // Load cache first
// if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
// return -1;
// }
// if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) {
// tsdbCloseMFile(pMFile);
// return -1;
// }
// while (true) {
// int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord));
// if (tsize == 0) break;
// if (tsize < 0) {
// tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno));
// return -1;
// }
// if (tsize < sizeof(SKVRecord)) {
// tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord),
// TSDB_FILE_FULL_NAME(pMFile));
// terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
// tsdbCloseMFile(pMFile);
// return -1;
// }
// void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo);
// ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord));
// // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true);
// if (rInfo.offset < 0) {
// taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid));
// #if 0
// pStore->info.size += sizeof(SKVRecord);
// pStore->info.nRecords--;
// pStore->info.nDels++;
// pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
// #endif
// } else {
// ASSERT(rInfo.offset > 0 && rInfo.size > 0);
// if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
// tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo),
// TSDB_FILE_FULL_NAME(pMFile));
// terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
// tsdbCloseMFile(pMFile);
// return -1;
// }
// maxBufSize = TMAX(maxBufSize, rInfo.size);
// if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) {
// tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
// tstrerror(terrno));
// tsdbCloseMFile(pMFile);
// return -1;
// }
// #if 0
// pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
// pStore->info.nRecords++;
// #endif
// }
// }
// if (recoverMeta) {
// pBuf = taosMemoryMalloc((size_t)maxBufSize);
// if (pBuf == NULL) {
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
// tsdbCloseMFile(pMFile);
// return -1;
// }
// SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL);
// while (pRecord) {
// if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
// tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
// tstrerror(terrno));
// taosMemoryFreeClear(pBuf);
// tsdbCloseMFile(pMFile);
// return -1;
// }
// int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
// if (nread < 0) {
// tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
// tstrerror(terrno));
// taosMemoryFreeClear(pBuf);
// tsdbCloseMFile(pMFile);
// return -1;
// }
// if (nread < pRecord->size) {
// tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
// REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
// terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
// taosMemoryFreeClear(pBuf);
// tsdbCloseMFile(pMFile);
// return -1;
// }
// if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) {
// tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid,
// tstrerror(terrno));
// taosMemoryFreeClear(pBuf);
// tsdbCloseMFile(pMFile);
// return -1;
// }
// pRecord = taosHashIterate(pfs->metaCache, pRecord);
// }
// tsdbOrgMeta(pRepo);
// }
// tsdbCloseMFile(pMFile);
// taosMemoryFreeClear(pBuf);
// return 0;
// }
static int tsdbScanRootDir(STsdb *pRepo) {
char rootDir[TSDB_FILENAME_LEN];
char bname[TSDB_FILENAME_LEN];
......@@ -983,127 +791,6 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) {
return false;
}
// static int tsdbRestoreMeta(STsdb *pRepo) {
// char rootDir[TSDB_FILENAME_LEN];
// char bname[TSDB_FILENAME_LEN];
// STfsDir * tdir = NULL;
// const STfsFile *pf = NULL;
// const char * pattern = "^meta(-ver[0-9]+)?$";
// regex_t regex;
// STsdbFS * pfs = REPO_FS(pRepo);
// regcomp(&regex, pattern, REG_EXTENDED);
// tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo));
// tsdbGetRootDir(REPO_ID(pRepo), rootDir);
// tdir = tfsOpendir(rootDir);
// if (tdir == NULL) {
// tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
// regfree(&regex);
// return -1;
// }
// while ((pf = tfsReaddir(tdir))) {
// tfsBasename(pf, bname);
// if (strcmp(bname, "data") == 0) {
// // Skip the data/ directory
// continue;
// }
// if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) {
// // Skip current.t file
// tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), pf->aname);
// (void)tfsremove(pf);
// continue;
// }
// int code = regexec(&regex, bname, 0, NULL, 0);
// if (code == 0) {
// // Match
// if (pfs->cstatus->pmf != NULL) {
// tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo),
// TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), pf->aname);
// terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
// tfsClosedir(tdir);
// regfree(&regex);
// return -1;
// } else {
// uint32_t _version = 0;
// if (strcmp(bname, "meta") != 0) {
// sscanf(bname, "meta-ver%" PRIu32, &_version);
// pfs->cstatus->meta.version = _version;
// }
// pfs->cstatus->pmf = &(pfs->cstatus->mf);
// pfs->cstatus->pmf->f = *pf;
// TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf);
// if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) {
// tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
// tfsClosedir(tdir);
// regfree(&regex);
// return -1;
// }
// if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) {
// tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
// tsdbCloseMFile(pfs->cstatus->pmf);
// tfsClosedir(tdir);
// regfree(&regex);
// return -1;
// }
// if (tsdbForceKeepFile) {
// struct stat tfstat;
// // Get real file size
// if (fstat(pfs->cstatus->pmf->fd, &tfstat) < 0) {
// terrno = TAOS_SYSTEM_ERROR(errno);
// tsdbCloseMFile(pfs->cstatus->pmf);
// tfsClosedir(tdir);
// regfree(&regex);
// return -1;
// }
// if (pfs->cstatus->pmf->info.size != tfstat.st_size) {
// int64_t tfsize = pfs->cstatus->pmf->info.size;
// pfs->cstatus->pmf->info.size = tfstat.st_size;
// tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
// TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), tfsize, pfs->cstatus->pmf->info.size);
// }
// }
// tsdbCloseMFile(pfs->cstatus->pmf);
// }
// } else if (code == REG_NOMATCH) {
// // Not match
// tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname);
// tfsremove(pf);
// continue;
// } else {
// // Has other error
// tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code));
// terrno = TAOS_SYSTEM_ERROR(code);
// tfsClosedir(tdir);
// regfree(&regex);
// return -1;
// }
// }
// if (pfs->cstatus->pmf) {
// tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf));
// } else {
// tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo));
// }
// tfsClosedir(tdir);
// regfree(&regex);
// return 0;
// }
static int tsdbRestoreDFileSet(STsdb *pRepo) {
char dataDir[TSDB_FILENAME_LEN];
char bname[TSDB_FILENAME_LEN];
......
source/dnode/vnode/src/tsdb/tsdbFile.c
浏览文件 @ 68f7c0ec
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
#include "tsdb.h"
static const char *TSDB_FNAME_SUFFIX[] = {
"head", // TSDB_FILE_HEAD
......@@ -33,271 +33,6 @@ static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
static int tsdbRollBackDFile(SDFile *pDFile);
#if 0
// ============== SMFile
void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
char fname[TSDB_FILENAME_LEN];
TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK);
memset(&(pMFile->info), 0, sizeof(pMFile->info));
pMFile->info.magic = TSDB_FILE_INIT_MAGIC;
tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname);
tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
}
void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) {
*pMFile = *pOMFile;
TSDB_FILE_SET_CLOSED(pMFile);
}
int tsdbEncodeSMFile(void **buf, SMFile *pMFile) {
int tlen = 0;
tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
tlen += tfsEncodeFile(buf, &(pMFile->f));
return tlen;
}
void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) {
buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
buf = tfsDecodeFile(buf, &(pMFile->f));
TSDB_FILE_SET_CLOSED(pMFile);
return buf;
}
int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) {
int tlen = 0;
tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile));
return tlen;
}
void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) {
char *aname;
buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
buf = taosDecodeString(buf, &aname);
strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN);
TSDB_FILE_SET_CLOSED(pMFile);
taosMemoryFreeClear(aname);
return buf;
}
int tsdbApplyMFileChange(SMFile *from, SMFile *to) {
if (from == NULL && to == NULL) return 0;
if (from != NULL) {
if (to == NULL) {
return tsdbRemoveMFile(from);
} else {
if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
if (from->info.size > to->info.size) {
tsdbRollBackMFile(to);
}
} else {
return tsdbRemoveMFile(from);
}
}
}
return 0;
}
int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) {
ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC);
pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pMFile->fd < 0) {
if (errno == ENOENT) {
// Try to create directory recursively
char *s = strdup(TFILE_REL_NAME(&(pMFile->f)));
if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) {
taosMemoryFreeClear(s);
return -1;
}
taosMemoryFreeClear(s);
pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pMFile->fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
} else {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
if (!updateHeader) {
return 0;
}
pMFile->info.size += TSDB_FILE_HEAD_SIZE;
if (tsdbUpdateMFileHeader(pMFile) < 0) {
tsdbCloseMFile(pMFile);
tsdbRemoveMFile(pMFile);
return -1;
}
return 0;
}
int tsdbUpdateMFileHeader(SMFile *pMFile) {
char buf[TSDB_FILE_HEAD_SIZE] = "\0";
if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
return -1;
}
void *ptr = buf;
tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile));
taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
return -1;
}
return 0;
}
int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) {
char buf[TSDB_FILE_HEAD_SIZE] = "\0";
ASSERT(TSDB_FILE_OPENED(pMFile));
if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
return -1;
}
if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
}
tsdbDecodeMFInfo(buf, pInfo);
return 0;
}
int tsdbScanAndTryFixMFile(STsdb *pRepo) {
SMFile * pMFile = pRepo->fs->cstatus->pmf;
struct stat mfstat;
SMFile mf;
if (pMFile == NULL) {
// No meta file, no need to scan
return 0;
}
tsdbInitMFileEx(&mf, pMFile);
if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) {
tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pMFile));
pRepo->state |= TSDB_STATE_BAD_META;
TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
return 0;
}
if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (pMFile->info.size < mfstat.st_size) {
if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
return -1;
}
if (taosFtruncate(mf.fd, mf.info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseMFile(&mf);
return -1;
}
if (tsdbUpdateMFileHeader(&mf) < 0) {
tsdbCloseMFile(&mf);
return -1;
}
tsdbCloseMFile(&mf);
tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
mfstat.st_size, pMFile->info.size);
} else if (pMFile->info.size > mfstat.st_size) {
tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size);
pRepo->state |= TSDB_STATE_BAD_META;
TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return 0;
} else {
tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile));
}
return 0;
}
int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) {
int tlen = 0;
tlen += taosEncodeVariantI64(buf, pInfo->size);
tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
tlen += taosEncodeVariantI64(buf, pInfo->nDels);
tlen += taosEncodeFixedU32(buf, pInfo->magic);
return tlen;
}
void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) {
buf = taosDecodeVariantI64(buf, &(pInfo->size));
buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
buf = taosDecodeFixedU32(buf, &(pInfo->magic));
return buf;
}
static int tsdbRollBackMFile(SMFile *pMFile) {
SMFile mf;
tsdbInitMFileEx(&mf, pMFile);
if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
return -1;
}
if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseMFile(&mf);
return -1;
}
if (tsdbUpdateMFileHeader(&mf) < 0) {
tsdbCloseMFile(&mf);
return -1;
}
TSDB_FILE_FSYNC(&mf);
tsdbCloseMFile(&mf);
return 0;
}
#endif
// ============== Operations on SDFile
void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype) {
char fname[TSDB_FILENAME_LEN];
......@@ -310,7 +45,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t
pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
pDFile->info.fver = tsdbGetDFSVersion(ftype);
tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname);
tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname);
tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
}
......
source/dnode/vnode/src/tsdb/tsdbMain.c 已删除 100644 → 0
浏览文件 @ d62ddcf2
此差异已折叠。
source/dnode/vnode/src/tsdb/tsdbOpen.c 0 → 100644
浏览文件 @ 68f7c0ec
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) {
STsdb *pTsdb = NULL;
int slen = 0;
*ppTsdb = NULL;
slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_TSDB_DIR) + 3;
// create handle
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
if (pTsdb == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pTsdb->path = (char *)&pTsdb[1];
sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP,
VNODE_TSDB_DIR);
pTsdb->pVnode = pVnode;
pTsdb->repoLocked = false;
tdbMutexInit(&pTsdb->mutex, NULL);
pTsdb->config = pVnode->config.tsdbCfg;
pTsdb->fs = tsdbNewFS(&pTsdb->config);
// create dir (TODO: use tfsMkdir)
taosMkDir(pTsdb->path);
// open tsdb
if (tsdbOpenFS(pTsdb) < 0) {
goto _err;
}
tsdbDebug("vgId: %d tsdb is opened", TD_VID(pVnode));
*ppTsdb = pTsdb;
return 0;
_err:
taosMemoryFree(pTsdb);
return -1;
}
int tsdbClose(STsdb *pTsdb) {
if (pTsdb) {
tsdbCloseFS(pTsdb);
tsdbFreeFS(pTsdb->fs);
taosMemoryFree(pTsdb);
}
return 0;
}
int tsdbLockRepo(STsdb *pTsdb) {
int code = taosThreadMutexLock(&pTsdb->mutex);
if (code != 0) {
tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
pTsdb->repoLocked = true;
return 0;
}
int tsdbUnlockRepo(STsdb *pTsdb) {
ASSERT(IS_REPO_LOCKED(pTsdb));
pTsdb->repoLocked = false;
int code = taosThreadMutexUnlock(&pTsdb->mutex);
if (code != 0) {
tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbOptions.c 已删除 100644 → 0
浏览文件 @ d62ddcf2
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
const STsdbCfg defautlTsdbOptions = {.precision = 0,
.lruCacheSize = 0,
.days = 10,
.minRows = 100,
.maxRows = 4096,
.keep2 = 3650,
.keep0 = 3650,
.keep1 = 3650,
.update = 0,
.compression = TWO_STAGE_COMP};
int tsdbValidateOptions(const STsdbCfg *pTsdbOptions) {
// TODO
return 0;
}
void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc) { memcpy(pDest, pSrc, sizeof(STsdbCfg)); }
\ No newline at end of file
source/libs/function/src/builtins.c
浏览文件 @ 68f7c0ec
此差异已折叠。
source/libs/function/src/tudf.c
浏览文件 @ 68f7c0ec
此差异已折叠。
source/libs/transport/src/trans.c
浏览文件 @ 68f7c0ec
此差异已折叠。
source/os/src/osMemory.c
浏览文件 @ 68f7c0ec
此差异已折叠。
source/util/src/tconfig.c
浏览文件 @ 68f7c0ec
此差异已折叠。
tests/pytest/util/dnodes.py
浏览文件 @ 68f7c0ec
此差异已折叠。
tests/script/jenkins/basic.txt
浏览文件 @ 68f7c0ec
此差异已折叠。
tests/script/sh/deploy.sh
浏览文件 @ 68f7c0ec
此差异已折叠。
tests/script/tsim/db/taosdlog.sim 0 → 100644
浏览文件 @ 68f7c0ec
此差异已折叠。
tests/system-test/0-others/taosShell.py 0 → 100644
浏览文件 @ 68f7c0ec
此差异已折叠。
tests/system-test/0-others/taosdlog.py 0 → 100644
浏览文件 @ 68f7c0ec
此差异已折叠。
tools/shell/inc/shellInt.h
浏览文件 @ 68f7c0ec
此差异已折叠。
tools/shell/src/shellEngine.c
浏览文件 @ 68f7c0ec
此差异已折叠。
tools/shell/src/shellMain.c
浏览文件 @ 68f7c0ec
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或