提交 c03f7aa7 编写于 作者: H Hongze Cheng

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/data_format

...@@ -46,7 +46,7 @@ ENDIF () ...@@ -46,7 +46,7 @@ ENDIF ()
IF (TD_WINDOWS) IF (TD_WINDOWS)
MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
SET(COMMON_FLAGS "/w /D_WIN32") SET(COMMON_FLAGS "/w /D_WIN32 /Zi")
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
# IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
# SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
......
...@@ -100,8 +100,10 @@ endif(${BUILD_WITH_NURAFT}) ...@@ -100,8 +100,10 @@ endif(${BUILD_WITH_NURAFT})
# addr2line # addr2line
if(${BUILD_ADDR2LINE}) if(${BUILD_ADDR2LINE})
if(NOT ${TD_WINDOWS})
cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(NOT ${TD_WINDOWS})
endif(${BUILD_ADDR2LINE}) endif(${BUILD_ADDR2LINE})
# download dependencies # download dependencies
...@@ -335,6 +337,7 @@ endif(${BUILD_WITH_SQLITE}) ...@@ -335,6 +337,7 @@ endif(${BUILD_WITH_SQLITE})
# addr2line # addr2line
if(${BUILD_ADDR2LINE}) if(${BUILD_ADDR2LINE})
if(NOT ${TD_WINDOWS})
check_include_file( "sys/types.h" HAVE_SYS_TYPES_H) check_include_file( "sys/types.h" HAVE_SYS_TYPES_H)
check_include_file( "sys/stat.h" HAVE_SYS_STAT_H ) check_include_file( "sys/stat.h" HAVE_SYS_STAT_H )
check_include_file( "inttypes.h" HAVE_INTTYPES_H ) check_include_file( "inttypes.h" HAVE_INTTYPES_H )
...@@ -374,6 +377,7 @@ if(${BUILD_ADDR2LINE}) ...@@ -374,6 +377,7 @@ if(${BUILD_ADDR2LINE})
add_library(addr2line STATIC "addr2line/addr2line.c") add_library(addr2line STATIC "addr2line/addr2line.c")
target_link_libraries(addr2line PUBLIC libdwarf dl z) target_link_libraries(addr2line PUBLIC libdwarf dl z)
target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" ) target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" )
endif(NOT ${TD_WINDOWS})
endif(${BUILD_ADDR2LINE}) endif(${BUILD_ADDR2LINE})
......
...@@ -18,7 +18,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" ...@@ -18,7 +18,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx"
`TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。 `TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。
`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动(taosc)建立与 TDengine 运行实例的连接,提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式,用户可以参考 [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) 文档自行编写。 `TDengine.Connector` 连接器支持通过 TDengine 客户端驱动(taosc)建立与 TDengine 运行实例的连接,提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式,用户可以参考 [REST API](/reference/rest-api/) 文档自行编写。
本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`,并通过 `TDengine.Connector` 连接 TDengine 集群,进行数据写入、查询等基本操作。 本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`,并通过 `TDengine.Connector` 连接 TDengine 集群,进行数据写入、查询等基本操作。
......
...@@ -19,7 +19,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" ...@@ -19,7 +19,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx"
`TDengine.Connector` is a C# language connector provided by TDengine that allows C# developers to develop C# applications that access TDengine cluster data. `TDengine.Connector` is a C# language connector provided by TDengine that allows C# developers to develop C# applications that access TDengine cluster data.
The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) documentation. The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [REST API](/reference/rest-api/) documentation.
This article describes how to install `TDengine.Connector` in a Linux or Windows environment and connect to TDengine clusters via `TDengine.Connector` to perform basic operations such as data writing and querying. This article describes how to install `TDengine.Connector` in a Linux or Windows environment and connect to TDengine clusters via `TDengine.Connector` to perform basic operations such as data writing and querying.
......
...@@ -244,12 +244,12 @@ typedef struct { ...@@ -244,12 +244,12 @@ typedef struct {
const void* pMsg; const void* pMsg;
} SSubmitMsgIter; } SSubmitMsgIter;
int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter);
int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
// for debug // for debug
int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema); int32_t tPrintFixedSchemaSubmitReq(SSubmitReq* pReq, STSchema* pSchema);
typedef struct { typedef struct {
int32_t code; int32_t code;
...@@ -1697,7 +1697,7 @@ int32_t tDecodeSRSmaParam(SDecoder* pCoder, SRSmaParam* pRSmaParam); ...@@ -1697,7 +1697,7 @@ int32_t tDecodeSRSmaParam(SDecoder* pCoder, SRSmaParam* pRSmaParam);
// TDMT_VND_CREATE_STB ============== // TDMT_VND_CREATE_STB ==============
typedef struct SVCreateStbReq { typedef struct SVCreateStbReq {
const char* name; char* name;
tb_uid_t suid; tb_uid_t suid;
int8_t rollup; int8_t rollup;
SSchemaWrapper schema; SSchemaWrapper schema;
...@@ -1710,7 +1710,7 @@ int tDecodeSVCreateStbReq(SDecoder* pCoder, SVCreateStbReq* pReq); ...@@ -1710,7 +1710,7 @@ int tDecodeSVCreateStbReq(SDecoder* pCoder, SVCreateStbReq* pReq);
// TDMT_VND_DROP_STB ============== // TDMT_VND_DROP_STB ==============
typedef struct SVDropStbReq { typedef struct SVDropStbReq {
const char* name; char* name;
tb_uid_t suid; tb_uid_t suid;
} SVDropStbReq; } SVDropStbReq;
...@@ -1723,13 +1723,13 @@ typedef struct SVCreateTbReq { ...@@ -1723,13 +1723,13 @@ typedef struct SVCreateTbReq {
int32_t flags; int32_t flags;
tb_uid_t uid; tb_uid_t uid;
int64_t ctime; int64_t ctime;
const char* name; char* name;
int32_t ttl; int32_t ttl;
int8_t type; int8_t type;
union { union {
struct { struct {
tb_uid_t suid; tb_uid_t suid;
const uint8_t* pTag; uint8_t* pTag;
} ctb; } ctb;
struct { struct {
SSchemaWrapper schema; SSchemaWrapper schema;
...@@ -1777,7 +1777,7 @@ int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatc ...@@ -1777,7 +1777,7 @@ int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatc
// TDMT_VND_DROP_TABLE ================= // TDMT_VND_DROP_TABLE =================
typedef struct { typedef struct {
const char* name; char* name;
int8_t igNotExists; int8_t igNotExists;
} SVDropTbReq; } SVDropTbReq;
...@@ -1809,9 +1809,9 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp); ...@@ -1809,9 +1809,9 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp);
// TDMT_VND_ALTER_TABLE ===================== // TDMT_VND_ALTER_TABLE =====================
typedef struct { typedef struct {
const char* tbName; char* tbName;
int8_t action; int8_t action;
const char* colName; char* colName;
// TSDB_ALTER_TABLE_ADD_COLUMN // TSDB_ALTER_TABLE_ADD_COLUMN
int8_t type; int8_t type;
int8_t flags; int8_t flags;
...@@ -1820,17 +1820,17 @@ typedef struct { ...@@ -1820,17 +1820,17 @@ typedef struct {
// TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES
int32_t colModBytes; int32_t colModBytes;
// TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME
const char* colNewName; char* colNewName;
// TSDB_ALTER_TABLE_UPDATE_TAG_VAL // TSDB_ALTER_TABLE_UPDATE_TAG_VAL
const char* tagName; char* tagName;
int8_t isNull; int8_t isNull;
uint32_t nTagVal; uint32_t nTagVal;
const uint8_t* pTagVal; uint8_t* pTagVal;
// TSDB_ALTER_TABLE_UPDATE_OPTIONS // TSDB_ALTER_TABLE_UPDATE_OPTIONS
int8_t updateTTL; int8_t updateTTL;
int32_t newTTL; int32_t newTTL;
int8_t updateComment; int8_t updateComment;
const char* newComment; char* newComment;
} SVAlterTbReq; } SVAlterTbReq;
int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
...@@ -2020,7 +2020,7 @@ static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { ...@@ -2020,7 +2020,7 @@ static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) {
int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp); int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp);
int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp); int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp);
void tFreeSClientHbBatchRsp(SClientHbBatchRsp *pBatchRsp); void tFreeSClientHbBatchRsp(SClientHbBatchRsp* pBatchRsp);
static FORCE_INLINE int32_t tEncodeSKv(SEncoder* pEncoder, const SKv* pKv) { static FORCE_INLINE int32_t tEncodeSKv(SEncoder* pEncoder, const SKv* pKv) {
if (tEncodeI32(pEncoder, pKv->key) < 0) return -1; if (tEncodeI32(pEncoder, pKv->key) < 0) return -1;
...@@ -2267,8 +2267,8 @@ typedef struct { ...@@ -2267,8 +2267,8 @@ typedef struct {
int64_t interval; int64_t interval;
int64_t offset; // use unit by precision of DB int64_t offset; // use unit by precision of DB
int64_t sliding; int64_t sliding;
const char* expr; // sma expression char* expr; // sma expression
const char* tagsFilter; char* tagsFilter;
} STSma; // Time-range-wise SMA } STSma; // Time-range-wise SMA
typedef STSma SVCreateTSmaReq; typedef STSma SVCreateTSmaReq;
...@@ -2600,7 +2600,7 @@ typedef struct { ...@@ -2600,7 +2600,7 @@ typedef struct {
int64_t uid; int64_t uid;
int32_t sver; int32_t sver;
uint32_t nData; uint32_t nData;
const uint8_t* pData; uint8_t* pData;
SVCreateTbReq cTbReq; SVCreateTbReq cTbReq;
} SVSubmitBlk; } SVSubmitBlk;
......
...@@ -46,24 +46,34 @@ typedef enum { ...@@ -46,24 +46,34 @@ typedef enum {
AUTH_TYPE_OTHER, AUTH_TYPE_OTHER,
} AUTH_TYPE; } AUTH_TYPE;
typedef struct SUserAuthInfo {
char user[TSDB_USER_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
AUTH_TYPE type;
} SUserAuthInfo;
typedef struct SCatalogReq { typedef struct SCatalogReq {
SArray *pTableName; // element is SNAME SArray *pTableMeta; // element is SNAME
SArray *pUdf; // udf name SArray *pDbVgroup; // element is db full name
SArray *pTableHash; // element is SNAME
SArray *pUdf; // element is udf name
SArray *pDbCfg; // element is db full name
SArray *pIndex; // element is index name
SArray *pUser; // element is SUserAuthInfo
bool qNodeRequired; // valid qnode bool qNodeRequired; // valid qnode
} SCatalogReq; } SCatalogReq;
typedef struct SMetaData { typedef struct SMetaData {
SArray *pTableMeta; // STableMeta array SArray *pTableMeta; // SArray<STableMeta>
SArray *pVgroupInfo; // SVgroupInfo list SArray *pDbVgroup; // SArray<SArray<SVgroupInfo>*>
SArray *pUdfList; // udf info list SArray *pTableHash; // SArray<SVgroupInfo>
SArray *pQnodeList; // qnode list, SArray<SQueryNodeAddr> SArray *pUdfList; // SArray<SFuncInfo>
SArray *pDbCfg; // SArray<SDbCfgInfo>
SArray *pIndex; // SArray<SIndexInfo>
SArray *pUser; // SArray<bool>
SArray *pQnodeList; // SArray<SQueryNodeAddr>
} SMetaData; } SMetaData;
typedef struct STbSVersion {
char* tbFName;
int32_t sver;
} STbSVersion;
typedef struct SCatalogCfg { typedef struct SCatalogCfg {
uint32_t maxTblCacheNum; uint32_t maxTblCacheNum;
uint32_t maxDBCacheNum; uint32_t maxDBCacheNum;
...@@ -88,6 +98,11 @@ typedef struct SDbVgVersion { ...@@ -88,6 +98,11 @@ typedef struct SDbVgVersion {
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
} SDbVgVersion; } SDbVgVersion;
typedef struct STbSVersion {
char* tbFName;
int32_t sver;
} STbSVersion;
typedef struct SUserAuthVersion { typedef struct SUserAuthVersion {
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
int32_t version; int32_t version;
...@@ -96,6 +111,8 @@ typedef struct SUserAuthVersion { ...@@ -96,6 +111,8 @@ typedef struct SUserAuthVersion {
typedef SDbCfgRsp SDbCfgInfo; typedef SDbCfgRsp SDbCfgInfo;
typedef SUserIndexRsp SIndexInfo; typedef SUserIndexRsp SIndexInfo;
typedef void (*catalogCallback)(SMetaData* pResult, void* param, int32_t code);
int32_t catalogInit(SCatalogCfg *cfg); int32_t catalogInit(SCatalogCfg *cfg);
/** /**
...@@ -131,7 +148,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t d ...@@ -131,7 +148,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t d
int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId);
int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName); int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName);
int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid);
...@@ -241,9 +258,9 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ ...@@ -241,9 +258,9 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_
int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg);
int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo);
int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass);
......
...@@ -39,7 +39,7 @@ typedef struct { ...@@ -39,7 +39,7 @@ typedef struct {
} SEncoder; } SEncoder;
typedef struct { typedef struct {
const uint8_t* data; uint8_t* data;
uint32_t size; uint32_t size;
uint32_t pos; uint32_t pos;
SCoderMem* mList; SCoderMem* mList;
...@@ -120,7 +120,7 @@ static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t le ...@@ -120,7 +120,7 @@ static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t le
static int32_t tEncodeCStr(SEncoder* pCoder, const char* val); static int32_t tEncodeCStr(SEncoder* pCoder, const char* val);
/* ------------------------ DECODE ------------------------ */ /* ------------------------ DECODE ------------------------ */
void tDecoderInit(SDecoder* pCoder, const uint8_t* data, uint32_t size); void tDecoderInit(SDecoder* pCoder, uint8_t* data, uint32_t size);
void tDecoderClear(SDecoder* SDecoder); void tDecoderClear(SDecoder* SDecoder);
int32_t tStartDecode(SDecoder* pCoder); int32_t tStartDecode(SDecoder* pCoder);
void tEndDecode(SDecoder* pCoder); void tEndDecode(SDecoder* pCoder);
...@@ -141,9 +141,9 @@ static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val); ...@@ -141,9 +141,9 @@ static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val);
static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val); static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val);
static int32_t tDecodeFloat(SDecoder* pCoder, float* val); static int32_t tDecodeFloat(SDecoder* pCoder, float* val);
static int32_t tDecodeDouble(SDecoder* pCoder, double* val); static int32_t tDecodeDouble(SDecoder* pCoder, double* val);
static int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len); static int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len);
static int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len); static int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len);
static int32_t tDecodeCStr(SDecoder* pCoder, const char** val); static int32_t tDecodeCStr(SDecoder* pCoder, char** val);
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val);
/* ------------------------ IMPL ------------------------ */ /* ------------------------ IMPL ------------------------ */
...@@ -317,7 +317,7 @@ static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { ...@@ -317,7 +317,7 @@ static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) {
if (tDecodeU16v(pCoder, &tval) < 0) { if (tDecodeU16v(pCoder, &tval) < 0) {
return -1; return -1;
} }
*val = ZIGZAGD(int16_t, tval); if (val) *val = ZIGZAGD(int16_t, tval);
return 0; return 0;
} }
...@@ -331,7 +331,7 @@ static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { ...@@ -331,7 +331,7 @@ static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) {
if (tDecodeU32v(pCoder, &tval) < 0) { if (tDecodeU32v(pCoder, &tval) < 0) {
return -1; return -1;
} }
*val = ZIGZAGD(int32_t, tval); if (val) *val = ZIGZAGD(int32_t, tval);
return 0; return 0;
} }
...@@ -345,7 +345,7 @@ static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { ...@@ -345,7 +345,7 @@ static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) {
if (tDecodeU64v(pCoder, &tval) < 0) { if (tDecodeU64v(pCoder, &tval) < 0) {
return -1; return -1;
} }
*val = ZIGZAGD(int64_t, tval); if (val) *val = ZIGZAGD(int64_t, tval);
return 0; return 0;
} }
...@@ -377,7 +377,7 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { ...@@ -377,7 +377,7 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) {
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len) { static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) {
if (tDecodeU32v(pCoder, len) < 0) return -1; if (tDecodeU32v(pCoder, len) < 0) return -1;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1;
...@@ -389,19 +389,19 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, ...@@ -389,19 +389,19 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val,
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len) { static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) {
if (tDecodeBinary(pCoder, (const uint8_t**)val, len) < 0) return -1; if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1;
(*len) -= 1; (*len) -= 1;
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, const char** val) { static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) {
uint32_t len; uint32_t len;
return tDecodeCStrAndLen(pCoder, val, &len); return tDecodeCStrAndLen(pCoder, val, &len);
} }
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
const char* pStr; char* pStr;
uint32_t len; uint32_t len;
if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1; if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1;
......
...@@ -6,16 +6,36 @@ ...@@ -6,16 +6,36 @@
set -e set -e
#set -x #set -x
verMode=edge
pagMode=full
iplist=""
serverFqdn=""
# -----------------------Variables definition--------------------- # -----------------------Variables definition---------------------
script_dir=$(dirname $(readlink -f "$0")) script_dir=$(dirname $(readlink -f "$0"))
# Dynamic directory # Dynamic directory
data_dir="/var/lib/taos"
log_dir="/var/log/taos"
data_link_dir="/usr/local/taos/data" clientName="taos"
log_link_dir="/usr/local/taos/log" serverName="taosd"
configFile="taos.cfg"
productName="TDengine"
emailName="taosdata.com"
uninstallScript="rmtaos"
historyFile="taos_history"
tarName="taos.tar.gz"
dataDir="/var/lib/taos"
logDir="/var/log/taos"
configDir="/etc/taos"
installDir="/usr/local/taos"
adapterName="taosadapter"
benchmarkName="taosBenchmark"
dumpName="taosdump"
demoName="taosdemo"
cfg_install_dir="/etc/taos" data_dir=${dataDir}
log_dir=${logDir}
cfg_install_dir=${configDir}
bin_link_dir="/usr/bin" bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib" lib_link_dir="/usr/lib"
...@@ -23,21 +43,13 @@ lib64_link_dir="/usr/lib64" ...@@ -23,21 +43,13 @@ lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include" inc_link_dir="/usr/include"
#install main path #install main path
install_main_dir="/usr/local/taos" install_main_dir=${installDir}
# old bin dir # old bin dir
bin_dir="/usr/local/taos/bin" bin_dir="${installDir}/bin"
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
nginx_port=6060
#taos-tools para nginx_dir="/usr/local/nginxd"
demoName="taosdemo"
benchmarkName="taosBenchmark"
dumpName="taosdump"
emailName="taosdata.com"
taosName="taos"
toolsName="taostools"
# Color setting # Color setting
RED='\033[0;31m' RED='\033[0;31m'
...@@ -47,8 +59,8 @@ GREEN_UNDERLINE='\033[4;32m' ...@@ -47,8 +59,8 @@ GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m' NC='\033[0m'
csudo="" csudo=""
if command -v sudo > /dev/null; then if command -v sudo >/dev/null; then
csudo="sudo" csudo="sudo "
fi fi
update_flag=0 update_flag=0
...@@ -56,16 +68,16 @@ prompt_force=0 ...@@ -56,16 +68,16 @@ prompt_force=0
initd_mod=0 initd_mod=0
service_mod=2 service_mod=2
if pidof systemd &> /dev/null; then if pidof systemd &>/dev/null; then
service_mod=0 service_mod=0
elif $(which service &> /dev/null); then elif $(which service &>/dev/null); then
service_mod=1 service_mod=1
service_config_dir="/etc/init.d" service_config_dir="/etc/init.d"
if $(which chkconfig &> /dev/null); then if $(which chkconfig &>/dev/null); then
initd_mod=1 initd_mod=1
elif $(which insserv &> /dev/null); then elif $(which insserv &>/dev/null); then
initd_mod=2 initd_mod=2
elif $(which update-rc.d &> /dev/null); then elif $(which update-rc.d &>/dev/null); then
initd_mod=3 initd_mod=3
else else
service_mod=2 service_mod=2
...@@ -74,34 +86,33 @@ else ...@@ -74,34 +86,33 @@ else
service_mod=2 service_mod=2
fi fi
# get the operating system type for using the corresponding init file # get the operating system type for using the corresponding init file
# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification
#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
if [[ -e /etc/os-release ]]; then if [[ -e /etc/os-release ]]; then
osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) || :
else else
osinfo="" osinfo=""
fi fi
#echo "osinfo: ${osinfo}" #echo "osinfo: ${osinfo}"
os_type=0 os_type=0
if echo $osinfo | grep -qwi "ubuntu" ; then if echo $osinfo | grep -qwi "ubuntu"; then
# echo "This is ubuntu system" # echo "This is ubuntu system"
os_type=1 os_type=1
elif echo $osinfo | grep -qwi "debian" ; then elif echo $osinfo | grep -qwi "debian"; then
# echo "This is debian system" # echo "This is debian system"
os_type=1 os_type=1
elif echo $osinfo | grep -qwi "Kylin" ; then elif echo $osinfo | grep -qwi "Kylin"; then
# echo "This is Kylin system" # echo "This is Kylin system"
os_type=1 os_type=1
elif echo $osinfo | grep -qwi "centos" ; then elif echo $osinfo | grep -qwi "centos"; then
# echo "This is centos system" # echo "This is centos system"
os_type=2 os_type=2
elif echo $osinfo | grep -qwi "fedora" ; then elif echo $osinfo | grep -qwi "fedora"; then
# echo "This is fedora system" # echo "This is fedora system"
os_type=2 os_type=2
elif echo $osinfo | grep -qwi "Linx" ; then elif echo $osinfo | grep -qwi "Linx"; then
# echo "This is Linx system" # echo "This is Linx system"
os_type=1 os_type=1
service_mod=0 service_mod=0
initd_mod=0 initd_mod=0
...@@ -110,11 +121,10 @@ else ...@@ -110,11 +121,10 @@ else
echo " osinfo: ${osinfo}" echo " osinfo: ${osinfo}"
echo " This is an officially unverified linux system," echo " This is an officially unverified linux system,"
echo " if there are any problems with the installation and operation, " echo " if there are any problems with the installation and operation, "
echo " please feel free to contact taosdata.com for support." echo " please feel free to contact ${emailName} for support."
os_type=1 os_type=1
fi fi
# ============================= get input parameters ================================================= # ============================= get input parameters =================================================
# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] # install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...]
...@@ -124,12 +134,11 @@ interactiveFqdn=yes # [yes | no] ...@@ -124,12 +134,11 @@ interactiveFqdn=yes # [yes | no]
verType=server # [server | client] verType=server # [server | client]
initType=systemd # [systemd | service | ...] initType=systemd # [systemd | service | ...]
while getopts "hv:e:i:" arg while getopts "hv:e:i:" arg; do
do
case $arg in case $arg in
e) e)
#echo "interactiveFqdn=$OPTARG" #echo "interactiveFqdn=$OPTARG"
interactiveFqdn=$( echo $OPTARG ) interactiveFqdn=$(echo $OPTARG)
;; ;;
v) v)
#echo "verType=$OPTARG" #echo "verType=$OPTARG"
...@@ -140,7 +149,7 @@ do ...@@ -140,7 +149,7 @@ do
initType=$(echo $OPTARG) initType=$(echo $OPTARG)
;; ;;
h) h)
echo "Usage: `basename $0` -v [server | client] -e [yes | no]" echo "Usage: $(basename $0) -v [server | client] -e [yes | no]"
exit 0 exit 0
;; ;;
?) #unknow option ?) #unknow option
...@@ -155,98 +164,163 @@ done ...@@ -155,98 +164,163 @@ done
function kill_process() { function kill_process() {
pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then if [ -n "$pid" ]; then
${csudo} kill -9 $pid || : ${csudo}kill -9 $pid || :
fi fi
} }
function install_main_path() { function install_main_path() {
#create install main dir and all sub dir #create install main dir and all sub dir
${csudo} rm -rf ${install_main_dir} || : ${csudo}rm -rf ${install_main_dir} || :
${csudo} mkdir -p ${install_main_dir} ${csudo}mkdir -p ${install_main_dir}
${csudo} mkdir -p ${install_main_dir}/cfg ${csudo}mkdir -p ${install_main_dir}/cfg
${csudo} mkdir -p ${install_main_dir}/bin ${csudo}mkdir -p ${install_main_dir}/bin
${csudo} mkdir -p ${install_main_dir}/connector # ${csudo}mkdir -p ${install_main_dir}/connector
${csudo} mkdir -p ${install_main_dir}/lib ${csudo}mkdir -p ${install_main_dir}/driver
${csudo} mkdir -p ${install_main_dir}/examples ${csudo}mkdir -p ${install_main_dir}/examples
${csudo} mkdir -p ${install_main_dir}/include ${csudo}mkdir -p ${install_main_dir}/include
${csudo} mkdir -p ${install_main_dir}/init.d # ${csudo}mkdir -p ${install_main_dir}/init.d
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
${csudo} mkdir -p ${nginx_dir} ${csudo}mkdir -p ${nginx_dir}
fi fi
if [[ -e ${script_dir}/email ]]; then if [[ -e ${script_dir}/email ]]; then
${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: ${csudo}cp ${script_dir}/email ${install_main_dir}/ || :
fi fi
} }
function install_bin() { function install_bin() {
# Remove links # Remove links
${csudo} rm -f ${bin_link_dir}/taos || : ${csudo}rm -f ${bin_link_dir}/${clientName} || :
${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo}rm -f ${bin_link_dir}/${serverName} || :
${csudo} rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/${adapterName} || :
${csudo} rm -f ${bin_link_dir}/create_table || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
${csudo} rm -f ${bin_link_dir}/tmq_sim || : ${csudo}rm -f ${bin_link_dir}/tarbitrator || :
${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo}rm -f ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || :
#${csudo} rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/*
#Make link #Make link
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
[ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
[ -x ${install_main_dir}/bin/create_table ] && ${csudo} ln -s ${install_main_dir}/bin/create_table ${bin_link_dir}/create_table || : [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || :
[ -x ${install_main_dir}/bin/tmq_sim ] && ${csudo} ln -s ${install_main_dir}/bin/tmq_sim ${bin_link_dir}/tmq_sim || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
# [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
# [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
# [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
[ -x ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ] && ${csudo}ln -s ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || :
[ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :
if [ "$verMode" == "cluster" ]; then
${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/*
${csudo}mkdir -p ${nginx_dir}/logs
${csudo}chmod 777 ${nginx_dir}/sbin/nginx
fi
} }
function install_lib() { function install_lib() {
# Remove links # Remove links
${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo} rm -f ${lib64_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
${csudo} rm -f ${lib_link_dir}/libtdb.* || : #${csudo}rm -rf ${v15_java_app_dir} || :
${csudo} rm -f ${lib64_link_dir}/libtdb.* || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/*
${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
fi fi
${csudo} ldconfig ${csudo}ldconfig
} }
function install_header() { function install_avro() {
${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : if [ "$osType" != "Darwin" ]; then
${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* avro_dir=${script_dir}/avro
${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h if [ -f "${avro_dir}/lib/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then
# ${csudo} ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}/usr/bin/install -c -d /usr/local/$1
${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}/usr/bin/install -c -m 755 ${avro_dir}/lib/libavro.so.23.0.0 /usr/local/$1
${csudo}ln -sf /usr/local/$1/libavro.so.23.0.0 /usr/local/$1/libavro.so.23
${csudo}ln -sf /usr/local/$1/libavro.so.23 /usr/local/$1/libavro.so
${csudo}/usr/bin/install -c -d /usr/local/$1
[ -f ${avro_dir}/lib/libavro.a ] &&
${csudo}/usr/bin/install -c -m 755 ${avro_dir}/lib/libavro.a /usr/local/$1
if [ -d /etc/ld.so.conf.d ]; then
echo "/usr/local/$1" | ${csudo}tee /etc/ld.so.conf.d/libavro.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/libavro.conf"
${csudo}ldconfig
else
echo "/etc/ld.so.conf.d not found!"
fi
fi
fi
} }
# temp install taosBenchmark function install_jemalloc() {
function install_taosTools() { jemalloc_dir=${script_dir}/jemalloc
${csudo} rm -f ${bin_link_dir}/${benchmarkName} || :
${csudo} rm -f ${bin_link_dir}/${dumpName} || :
${csudo} rm -f ${bin_link_dir}/rm${toolsName} || :
${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${dumpName} ${install_main_dir}/bin/${dumpName} if [ -d ${jemalloc_dir} ]; then
${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${benchmarkName} ${csudo}/usr/bin/install -c -d /usr/local/bin
${csudo} ln -sf ${install_main_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${demoName}
#Make link if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then
[[ -x ${install_main_dir}/bin/${benchmarkName} ]] && \ ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin
${csudo} ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : fi
[[ -x ${install_main_dir}/bin/${demoName} ]] && \ if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then
${csudo} ln -s ${install_main_dir}/bin/${demoName} ${bin_link_dir}/${demoName} || : ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin
[[ -x ${install_main_dir}/bin/${dumpName} ]] && \ fi
${csudo} ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : if [ -f ${jemalloc_dir}/bin/jeprof ]; then
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin
fi
if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then
${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc
fi
if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then
${csudo}/usr/bin/install -c -d /usr/local/lib
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib
${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
${csudo}/usr/bin/install -c -d /usr/local/lib
if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
fi
if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
fi
if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
fi
fi
if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then
${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc
fi
if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then
${csudo}/usr/bin/install -c -d /usr/local/share/man/man3
${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
fi
if [ -d /etc/ld.so.conf.d ]; then
echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf"
${csudo}ldconfig
else
echo "/etc/ld.so.conf.d not found!"
fi
fi
}
function install_header() {
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || :
${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*
${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
} }
function add_newHostname_to_hosts() { function add_newHostname_to_hosts() {
...@@ -256,13 +330,12 @@ function add_newHostname_to_hosts() { ...@@ -256,13 +330,12 @@ function add_newHostname_to_hosts() {
iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
arr=($iphost) arr=($iphost)
IFS="$OLD_IFS" IFS="$OLD_IFS"
for s in "${arr[@]}" for s in "${arr[@]}"; do
do
if [[ "$s" == "$localIp" ]]; then if [[ "$s" == "$localIp" ]]; then
return return
fi fi
done done
${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: ${csudo}echo "127.0.0.1 $1" >>/etc/hosts || :
} }
function set_hostname() { function set_hostname() {
...@@ -276,28 +349,25 @@ function set_hostname() { ...@@ -276,28 +349,25 @@ function set_hostname() {
fi fi
done done
${csudo} hostname $newHostname ||: ${csudo}hostname $newHostname || :
retval=`echo $?` retval=$(echo $?)
if [[ $retval != 0 ]]; then if [[ $retval != 0 ]]; then
echo echo
echo "set hostname fail!" echo "set hostname fail!"
return return
fi fi
#echo -e -n "$(hostnamectl status --static)"
#echo -e -n "$(hostnamectl status --transient)"
#echo -e -n "$(hostnamectl status --pretty)"
#ubuntu/centos /etc/hostname #ubuntu/centos /etc/hostname
if [[ -e /etc/hostname ]]; then if [[ -e /etc/hostname ]]; then
${csudo} echo $newHostname > /etc/hostname ||: ${csudo}echo $newHostname >/etc/hostname || :
fi fi
#debian: #HOSTNAME=yourname #debian: #HOSTNAME=yourname
if [[ -e /etc/sysconfig/network ]]; then if [[ -e /etc/sysconfig/network ]]; then
${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || :
fi fi
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile}
serverFqdn=$newHostname serverFqdn=$newHostname
if [[ -e /etc/hosts ]]; then if [[ -e /etc/hosts ]]; then
...@@ -311,8 +381,7 @@ function is_correct_ipaddr() { ...@@ -311,8 +381,7 @@ function is_correct_ipaddr() {
IFS=" " IFS=" "
arr=($iplist) arr=($iplist)
IFS="$OLD_IFS" IFS="$OLD_IFS"
for s in "${arr[@]}" for s in "${arr[@]}"; do
do
if [[ "$s" == "$newIp" ]]; then if [[ "$s" == "$newIp" ]]; then
return 0 return 0
fi fi
...@@ -322,9 +391,9 @@ function is_correct_ipaddr() { ...@@ -322,9 +391,9 @@ function is_correct_ipaddr() {
} }
function set_ipAsFqdn() { function set_ipAsFqdn() {
iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: iplist=$(ip address | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}' | awk -F "/" '{print $1}') || :
if [ -z "$iplist" ]; then if [ -z "$iplist" ]; then
iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: iplist=$(ifconfig | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}' | awk -F ":" '{print $2}') || :
fi fi
if [ -z "$iplist" ]; then if [ -z "$iplist" ]; then
...@@ -332,7 +401,7 @@ function set_ipAsFqdn() { ...@@ -332,7 +401,7 @@ function set_ipAsFqdn() {
echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}"
localFqdn="127.0.0.1" localFqdn="127.0.0.1"
# Write the local FQDN to configuration file # Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile}
serverFqdn=$localFqdn serverFqdn=$localFqdn
echo echo
return return
...@@ -349,12 +418,12 @@ function set_ipAsFqdn() { ...@@ -349,12 +418,12 @@ function set_ipAsFqdn() {
if [ ! -z "$localFqdn" ]; then if [ ! -z "$localFqdn" ]; then
# Check if correct ip address # Check if correct ip address
is_correct_ipaddr $localFqdn is_correct_ipaddr $localFqdn
retval=`echo $?` retval=$(echo $?)
if [[ $retval != 0 ]]; then if [[ $retval != 0 ]]; then
read -p "Please choose an IP from local IP list:" localFqdn read -p "Please choose an IP from local IP list:" localFqdn
else else
# Write the local FQDN to configuration file # Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile}
serverFqdn=$localFqdn serverFqdn=$localFqdn
break break
fi fi
...@@ -373,20 +442,19 @@ function local_fqdn_check() { ...@@ -373,20 +442,19 @@ function local_fqdn_check() {
echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}"
echo echo
while true while true; do
do
read -r -p "Set hostname now? [Y/n] " input read -r -p "Set hostname now? [Y/n] " input
if [ ! -n "$input" ]; then if [ ! -n "$input" ]; then
set_hostname set_hostname
break break
else else
case $input in case $input in
[yY][eE][sS]|[yY]) [yY][eE][sS] | [yY])
set_hostname set_hostname
break break
;; ;;
[nN][oO]|[nN]) [nN][oO] | [nN])
set_ipAsFqdn set_ipAsFqdn
break break
;; ;;
...@@ -400,174 +468,523 @@ function local_fqdn_check() { ...@@ -400,174 +468,523 @@ function local_fqdn_check() {
fi fi
} }
function install_adapter_config() {
if [ ! -f "${cfg_install_dir}/${adapterName}.toml" ]; then
${csudo}mkdir -p ${cfg_install_dir}
[ -f ${script_dir}/cfg/${adapterName}.toml ] && ${csudo}cp ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}
[ -f ${cfg_install_dir}/${adapterName}.toml ] && ${csudo}chmod 644 ${cfg_install_dir}/${adapterName}.toml
fi
[ -f ${script_dir}/cfg/${adapterName}.toml ] &&
${csudo}cp -f ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}/${adapterName}.toml.new
[ -f ${cfg_install_dir}/${adapterName}.toml ] &&
${csudo}ln -s ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml
[ ! -z $1 ] && return 0 || : # only install client
}
function install_config() {
if [ ! -f "${cfg_install_dir}/${configFile}" ]; then
${csudo}mkdir -p ${cfg_install_dir}
[ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir}
${csudo}chmod 644 ${cfg_install_dir}/*
fi
${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new
${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg
[ ! -z $1 ] && return 0 || : # only install client
if ((${update_flag} == 1)); then
return 0
fi
if [ "$interactiveFqdn" == "no" ]; then
return 0
fi
local_fqdn_check
echo
echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName}:6030) of an existing ${productName} cluster node to join${NC}"
echo
echo -e -n "${GREEN}OR leave it blank to build one${NC}:"
read firstEp
while true; do
if [ ! -z "$firstEp" ]; then
${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile}
break
else
break
fi
done
echo
echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: "
read emailAddr
while true; do
if [ ! -z "$emailAddr" ]; then
email_file="${install_main_dir}/email"
${csudo}bash -c "echo $emailAddr > ${email_file}"
break
else
break
fi
done
}
function install_log() { function install_log() {
${csudo} rm -rf ${log_dir} || : ${csudo}rm -rf ${log_dir} || :
${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
${csudo} ln -s ${log_dir} ${install_main_dir}/log ${csudo}ln -s ${log_dir} ${install_main_dir}/log
} }
function install_data() { function install_data() {
${csudo} mkdir -p ${data_dir} ${csudo}mkdir -p ${data_dir}
${csudo}ln -s ${data_dir} ${install_main_dir}/data
}
function install_connector() {
[ -d "${script_dir}/connector/" ] && ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/
}
function install_examples() {
if [ -d ${script_dir}/examples ]; then
${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples
fi
}
function clean_service_on_sysvinit() {
if pidof ${serverName} &>/dev/null; then
${csudo}service ${serverName} stop || :
fi
if pidof tarbitrator &>/dev/null; then
${csudo}service tarbitratord stop || :
fi
${csudo} ln -s ${data_dir} ${install_main_dir}/data if ((${initd_mod} == 1)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}chkconfig --del ${serverName} || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}chkconfig --del tarbitratord || :
fi
elif ((${initd_mod} == 2)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}insserv -r ${serverName} || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}insserv -r tarbitratord || :
fi
elif ((${initd_mod} == 3)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}update-rc.d -f ${serverName} remove || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}update-rc.d -f tarbitratord remove || :
fi
fi
${csudo}rm -f ${service_config_dir}/${serverName} || :
${csudo}rm -f ${service_config_dir}/tarbitratord || :
if $(which init &>/dev/null); then
${csudo}init q || :
fi
}
function install_service_on_sysvinit() {
clean_service_on_sysvinit
sleep 1
if ((${os_type} == 1)); then
# ${csudo}cp -f ${script_dir}/init.d/${serverName}.deb ${install_main_dir}/init.d/${serverName}
${csudo}cp ${script_dir}/init.d/${serverName}.deb ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName}
# ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord
${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord
elif ((${os_type} == 2)); then
# ${csudo}cp -f ${script_dir}/init.d/${serverName}.rpm ${install_main_dir}/init.d/${serverName}
${csudo}cp ${script_dir}/init.d/${serverName}.rpm ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName}
# ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord
${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord
fi
if ((${initd_mod} == 1)); then
${csudo}chkconfig --add ${serverName} || :
${csudo}chkconfig --level 2345 ${serverName} on || :
${csudo}chkconfig --add tarbitratord || :
${csudo}chkconfig --level 2345 tarbitratord on || :
elif ((${initd_mod} == 2)); then
${csudo}insserv ${serverName} || :
${csudo}insserv -d ${serverName} || :
${csudo}insserv tarbitratord || :
${csudo}insserv -d tarbitratord || :
elif ((${initd_mod} == 3)); then
${csudo}update-rc.d ${serverName} defaults || :
${csudo}update-rc.d tarbitratord defaults || :
fi
} }
function clean_service_on_systemd() { function clean_service_on_systemd() {
taosd_service_config="${service_config_dir}/taosd.service" taosd_service_config="${service_config_dir}/${serverName}.service"
if systemctl is-active --quiet taosd; then if systemctl is-active --quiet ${serverName}; then
echo "TDengine is running, stopping it..." echo "${productName} is running, stopping it..."
${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null ${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null
fi fi
${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null ${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null
${csudo} rm -f ${taosd_service_config} ${csudo}rm -f ${taosd_service_config}
tarbitratord_service_config="${service_config_dir}/tarbitratord.service" tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
if systemctl is-active --quiet tarbitratord; then if systemctl is-active --quiet tarbitratord; then
echo "tarbitrator is running, stopping it..." echo "tarbitrator is running, stopping it..."
${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null ${csudo}systemctl stop tarbitratord &>/dev/null || echo &>/dev/null
fi fi
${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null
${csudo} rm -f ${tarbitratord_service_config} ${csudo}rm -f ${tarbitratord_service_config}
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
nginx_service_config="${service_config_dir}/nginxd.service" nginx_service_config="${service_config_dir}/nginxd.service"
if systemctl is-active --quiet nginxd; then if systemctl is-active --quiet nginxd; then
echo "Nginx for TDengine is running, stopping it..." echo "Nginx for ${productName} is running, stopping it..."
${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null ${csudo}systemctl stop nginxd &>/dev/null || echo &>/dev/null
fi fi
${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null ${csudo}systemctl disable nginxd &>/dev/null || echo &>/dev/null
${csudo} rm -f ${nginx_service_config} ${csudo}rm -f ${nginx_service_config}
fi fi
} }
# taos:2345:respawn:/etc/init.d/taosd start
function install_service_on_systemd() { function install_service_on_systemd() {
clean_service_on_systemd clean_service_on_systemd
taosd_service_config="${service_config_dir}/taosd.service" [ -f ${script_dir}/cfg/${serverName}.service ] &&
${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}" ${csudo}cp ${script_dir}/cfg/${serverName}.service \
${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" ${service_config_dir}/ || :
${csudo} bash -c "echo 'After=network-online.target taosadapter.service' >> ${taosd_service_config}" ${csudo}systemctl daemon-reload
${csudo} bash -c "echo 'Wants=network-online.target taosadapter.service' >> ${taosd_service_config}"
${csudo} bash -c "echo >> ${taosd_service_config}" ${csudo}systemctl enable ${serverName}
${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}"
${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" [ -f ${script_dir}/cfg/tarbitratord.service ] &&
${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" ${csudo}cp ${script_dir}/cfg/tarbitratord.service \
${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" ${service_config_dir}/ || :
${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" ${csudo}systemctl daemon-reload
${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}"
${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" if [ "$verMode" == "cluster" ]; then
${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" [ -f ${script_dir}/cfg/nginxd.service ] &&
${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}" ${csudo}cp ${script_dir}/cfg/nginxd.service \
${csudo} bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}" ${service_config_dir}/ || :
${csudo} bash -c "echo 'Restart=always' >> ${taosd_service_config}" ${csudo}systemctl daemon-reload
${csudo} bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}"
${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" if ! ${csudo}systemctl enable nginxd &>/dev/null; then
#${csudo} bash -c "echo 'StartLimitIntervalSec=60s' >> ${taosd_service_config}" ${csudo}systemctl daemon-reexec
${csudo} bash -c "echo >> ${taosd_service_config}" ${csudo}systemctl enable nginxd
${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" fi
${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" ${csudo}systemctl start nginxd
${csudo} systemctl enable taosd fi
}
${csudo} systemctl daemon-reload
function install_adapter_service() {
if ((${service_mod} == 0)); then
[ -f ${script_dir}/cfg/${adapterName}.service ] &&
${csudo}cp ${script_dir}/cfg/${adapterName}.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
fi
} }
function install_service() { function install_service() {
# if ((${service_mod}==0)); then if ((${service_mod} == 0)); then
# install_service_on_systemd install_service_on_systemd
# elif ((${service_mod}==1)); then elif ((${service_mod} == 1)); then
# install_service_on_sysvinit install_service_on_sysvinit
# else else
# # must manual stop taosd kill_process ${serverName}
kill_process taosd fi
# fi
} }
function install_config() { vercomp() {
if [ ! -f ${cfg_install_dir}/${configFile} ]; then if [[ $1 == $2 ]]; then
${csudo}mkdir -p ${cfg_install_dir} return 0
[ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir}
${csudo}chmod 644 ${cfg_install_dir}/*
fi fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do
ver1[i]=0
done
${csudo}cp -f ${script_dir}/cfg/${configFile} ${install_main_dir}/cfg/${configFile}.org for ((i = 0; i < ${#ver1[@]}; i++)); do
${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg if [[ -z ${ver2[i]} ]]; then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]})); then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]})); then
return 2
fi
done
return 0
}
function is_version_compatible() {
curr_version=$(ls ${script_dir}/driver/libtaos.so* | awk -F 'libtaos.so.' '{print $2}')
if [ -f ${script_dir}/driver/vercomp.txt ]; then
min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt)
else
min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5)
fi
exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3)
vercomp $exist_version "2.0.16.0"
case $? in
2)
prompt_force=1
;;
esac
vercomp $curr_version $min_compatible_version
echo "" # avoid $? value not update
case $? in
0) return 0 ;;
1) return 0 ;;
2) return 1 ;;
esac
}
function updateProduct() {
# Check if version compatible
if ! is_version_compatible; then
echo -e "${RED}Version incompatible${NC}"
return 1
fi
# Start to update
if [ ! -e ${tarName} ]; then
echo "File ${tarName} does not exist"
exit 1
fi
tar -zxf ${tarName}
install_jemalloc
echo -e "${GREEN}Start to update ${productName}...${NC}"
# Stop the service if running
if pidof ${serverName} &>/dev/null; then
if ((${service_mod} == 0)); then
${csudo}systemctl stop ${serverName} || :
elif ((${service_mod} == 1)); then
${csudo}service ${serverName} stop || :
else
kill_process ${serverName}
fi
sleep 1
fi
if [ "$verMode" == "cluster" ]; then
if pidof nginx &>/dev/null; then
if ((${service_mod} == 0)); then
${csudo}systemctl stop nginxd || :
elif ((${service_mod} == 1)); then
${csudo}service nginxd stop || :
else
kill_process nginx
fi
sleep 1
fi
fi
install_main_path
install_log
install_header
install_lib
if [ "$verMode" == "cluster" ]; then
install_connector
fi
install_examples
if [ -z $1 ]; then
install_bin
install_service
install_adapter_service
install_config
install_adapter_config
openresty_work=false
if [ "$verMode" == "cluster" ]; then
# Check if openresty is installed
# Check if nginx is installed successfully
if type curl &>/dev/null; then
if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then
echo -e "\033[44;32;1mNginx for ${productName} is updated successfully!${NC}"
openresty_work=true
else
echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m"
fi
fi
fi
echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}"
echo -e "${GREEN_DARK}To configure Adapter (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml"
if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
else
echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}"
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}"
fi
if [ ${openresty_work} = 'true' ]; then
echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
else
echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}"
fi
if ((${prompt_force} == 1)); then
echo ""
echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName} $exist_version!${NC}"
fi
echo
echo -e "\033[44;32;1m${productName} is updated successfully!${NC}"
else
install_bin
install_config
echo
echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}"
fi
rm -rf $(tar -tf ${tarName} | grep -v "^\./$")
} }
function install_TDengine() { function installProduct() {
# Start to install # Start to install
echo -e "${GREEN}Start to install TDengine...${NC}" if [ ! -e ${tarName} ]; then
echo "File ${tarName} does not exist"
exit 1
fi
tar -zxf ${tarName}
echo -e "${GREEN}Start to install ${productName}...${NC}"
install_main_path install_main_path
if [ -z $1 ]; then
install_data install_data
fi
install_log install_log
install_header install_header
install_lib install_lib
install_taosTools install_jemalloc
#install_avro lib
#install_avro lib64
if [ "$verMode" == "cluster" ]; then
install_connector
fi
install_examples
if [ -z $1 ]; then # install service and client if [ -z $1 ]; then # install service and client
# For installing new # For installing new
install_bin install_bin
install_service install_service
install_adapter_service
install_adapter_config
openresty_work=false
if [ "$verMode" == "cluster" ]; then
# Check if nginx is installed successfully
if type curl &>/dev/null; then
if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then
echo -e "\033[44;32;1mNginx for ${productName} is installed successfully!${NC}"
openresty_work=true
else
echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m"
fi
fi
fi
install_config install_config
# Ask if to start the service # Ask if to start the service
#echo
#echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
echo echo
echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}"
if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To configure ${adapterName} (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml"
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}" if ((${service_mod} == 0)); then
elif ((${service_mod}==1)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}" elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
else else
echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}" echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}"
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}"
fi fi
if [ ! -z "$firstEp" ]; then if [ ! -z "$firstEp" ]; then
tmpFqdn=${firstEp%%:*} tmpFqdn=${firstEp%%:*}
substr=":" substr=":"
if [[ $firstEp =~ $substr ]];then if [[ $firstEp =~ $substr ]]; then
tmpPort=${firstEp#*:} tmpPort=${firstEp#*:}
else else
tmpPort="" tmpPort=""
fi fi
if [[ "$tmpPort" != "" ]];then if [[ "$tmpPort" != "" ]]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else else
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo echo
elif [ ! -z "$serverFqdn" ]; then elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $serverFqdn${GREEN_DARK} to login into ${productName} server${NC}"
echo echo
fi fi
echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo -e "\033[44;32;1m${productName} is installed successfully!${NC}"
echo echo
else # Only install client else # Only install client
install_bin install_bin
install_config install_config
echo echo
echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}"
fi fi
touch ~/.taos_history touch ~/.${historyFile}
rm -rf $(tar -tf ${tarName} | grep -v "^\./$")
} }
## ==============================Main program starts from here============================ ## ==============================Main program starts from here============================
serverFqdn=$(hostname) serverFqdn=$(hostname)
if [ "$verType" == "server" ]; then if [ "$verType" == "server" ]; then
# Install server and client # Install server and client
install_TDengine if [ -x ${bin_dir}/${serverName} ]; then
update_flag=1
updateProduct
else
installProduct
fi
elif [ "$verType" == "client" ]; then elif [ "$verType" == "client" ]; then
interactiveFqdn=no interactiveFqdn=no
# Only install client # Only install client
install_TDengine client if [ -x ${bin_dir}/${clientName} ]; then
update_flag=1
updateProduct client
else
installProduct client
fi
else else
echo "please input correct verType" echo "please input correct verType"
fi fi
...@@ -121,7 +121,7 @@ struct SAppInstInfo { ...@@ -121,7 +121,7 @@ struct SAppInstInfo {
SCorEpSet mgmtEp; SCorEpSet mgmtEp;
SInstanceSummary summary; SInstanceSummary summary;
SList* pConnList; // STscObj linked list SList* pConnList; // STscObj linked list
int64_t clusterId; uint64_t clusterId;
void* pTransporter; void* pTransporter;
SAppHbMgr* pAppHbMgr; SAppHbMgr* pAppHbMgr;
}; };
...@@ -286,6 +286,8 @@ void initMsgHandleFp(); ...@@ -286,6 +286,8 @@ void initMsgHandleFp();
TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port, int connType); uint16_t port, int connType);
SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen);
int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb);
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
......
...@@ -565,10 +565,32 @@ const char *taos_get_server_info(TAOS *taos) { ...@@ -565,10 +565,32 @@ const char *taos_get_server_info(TAOS *taos) {
void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
if (taos == NULL || sql == NULL) { if (taos == NULL || sql == NULL) {
// todo directly call fp fp(param, NULL, TSDB_CODE_INVALID_PARA);
return;
}
SRequestObj* pRequest = NULL;
int32_t retryNum = 0;
int32_t code = 0;
size_t sqlLen = strlen(sql);
while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
pRequest = launchQuery(taos, sql, sqlLen);
if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
break;
}
code = refreshMeta(taos, pRequest);
if (code) {
pRequest->code = code;
break;
}
destroyRequest(pRequest);
} }
taos_query_l(taos, sql, (int32_t)strlen(sql)); fp(param, pRequest, code);
} }
void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
......
...@@ -125,10 +125,10 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { ...@@ -125,10 +125,10 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
struct SCatalog* pCatalog = NULL; struct SCatalog* pCatalog = NULL;
if (usedbRsp.vgVersion >= 0) { if (usedbRsp.vgVersion >= 0) {
int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId;
int32_t code1 = catalogGetHandle(clusterId, &pCatalog);
if (code1 != TSDB_CODE_SUCCESS) { if (code1 != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tscWarn("0x%" PRIx64 "catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->requestId, clusterId, tstrerror(code1));
tstrerror(code1));
} else { } else {
catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid); catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid);
} }
...@@ -158,7 +158,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { ...@@ -158,7 +158,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash); if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash);
taosMemoryFreeClear(output.dbVgroup); taosMemoryFreeClear(output.dbVgroup);
tscError("failed to build use db output since %s", terrstr()); tscError("0x%" PRIx64" failed to build use db output since %s", pRequest->requestId, terrstr());
} else if (output.dbVgroup) { } else if (output.dbVgroup) {
struct SCatalog* pCatalog = NULL; struct SCatalog* pCatalog = NULL;
......
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
#undef TD_MSG_SEG_CODE_ #undef TD_MSG_SEG_CODE_
#include "tmsgdef.h" #include "tmsgdef.h"
int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
if (pMsg == NULL) { if (pMsg == NULL) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
return -1; return -1;
...@@ -102,7 +102,7 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { ...@@ -102,7 +102,7 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
} }
} }
int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq *pReq, STSchema *pTschema) { int32_t tPrintFixedSchemaSubmitReq(SSubmitReq *pReq, STSchema *pTschema) {
SSubmitMsgIter msgIter = {0}; SSubmitMsgIter msgIter = {0};
if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1; if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1;
while (true) { while (true) {
......
...@@ -113,6 +113,8 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO ...@@ -113,6 +113,8 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
SRpcMsg rsp = {.info = pMsg->info}; SRpcMsg rsp = {.info = pMsg->info};
vnodePreprocessReq(pVnode->pImpl, pMsg);
int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false); int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false);
if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId); dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId);
......
...@@ -51,7 +51,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); ...@@ -51,7 +51,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs);
void vnodeDestroy(const char *path, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs);
SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb);
void vnodeClose(SVnode *pVnode); void vnodeClose(SVnode *pVnode);
int32_t vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version); int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg);
int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp); int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp);
int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
...@@ -126,7 +126,7 @@ STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); ...@@ -126,7 +126,7 @@ STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta);
void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList); void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList);
int32_t tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
int32_t tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
int32_t tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList); int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlock(STqReadHandle *pHandle);
...@@ -177,7 +177,7 @@ struct SMetaEntry { ...@@ -177,7 +177,7 @@ struct SMetaEntry {
int64_t version; int64_t version;
int8_t type; int8_t type;
tb_uid_t uid; tb_uid_t uid;
const char *name; char *name;
union { union {
struct { struct {
SSchemaWrapper schema; SSchemaWrapper schema;
...@@ -187,7 +187,7 @@ struct SMetaEntry { ...@@ -187,7 +187,7 @@ struct SMetaEntry {
int64_t ctime; int64_t ctime;
int32_t ttlDays; int32_t ttlDays;
tb_uid_t suid; tb_uid_t suid;
const uint8_t *pTags; uint8_t *pTags;
} ctbEntry; } ctbEntry;
struct { struct {
int64_t ctime; int64_t ctime;
......
...@@ -104,7 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep ...@@ -104,7 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep
int tsdbClose(STsdb** pTsdb); int tsdbClose(STsdb** pTsdb);
int tsdbBegin(STsdb* pTsdb); int tsdbBegin(STsdb* pTsdb);
int tsdbCommit(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, const SSubmitReq* pMsg); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp);
tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
......
...@@ -30,9 +30,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ...@@ -30,9 +30,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
int vLen = 0; int vLen = 0;
const void *pKey = NULL; const void *pKey = NULL;
const void *pVal = NULL; const void *pVal = NULL;
void * pBuf = NULL; void *pBuf = NULL;
int32_t szBuf = 0; int32_t szBuf = 0;
void * p = NULL; void *p = NULL;
SMetaReader mr = {0}; SMetaReader mr = {0};
// validate req // validate req
...@@ -71,9 +71,9 @@ _err: ...@@ -71,9 +71,9 @@ _err:
} }
int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
TBC * pNameIdxc = NULL; TBC *pNameIdxc = NULL;
TBC * pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TBC * pCtbIdxc = NULL; TBC *pCtbIdxc = NULL;
SCtbIdxKey *pCtbIdxKey; SCtbIdxKey *pCtbIdxKey;
const void *pKey = NULL; const void *pKey = NULL;
int nKey; int nKey;
...@@ -134,8 +134,8 @@ _err: ...@@ -134,8 +134,8 @@ _err:
int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaEntry oStbEntry = {0}; SMetaEntry oStbEntry = {0};
SMetaEntry nStbEntry = {0}; SMetaEntry nStbEntry = {0};
TBC * pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TBC * pTbDbc = NULL; TBC *pTbDbc = NULL;
const void *pData; const void *pData;
int nData; int nData;
int64_t oversion; int64_t oversion;
...@@ -165,7 +165,9 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ...@@ -165,7 +165,9 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
ASSERT(ret == 0); ASSERT(ret == 0);
tDecoderInit(&dc, pData, nData); oStbEntry.pBuf = taosMemoryMalloc(nData);
memcpy(oStbEntry.pBuf, pData, nData);
tDecoderInit(&dc, oStbEntry.pBuf, nData);
metaDecodeEntry(&dc, &oStbEntry); metaDecodeEntry(&dc, &oStbEntry);
nStbEntry.version = version; nStbEntry.version = version;
...@@ -193,6 +195,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ...@@ -193,6 +195,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
// update uid index // update uid index
tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0);
if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
metaULock(pMeta); metaULock(pMeta);
tDecoderClear(&dc); tDecoderClear(&dc);
tdbTbcClose(pTbDbc); tdbTbcClose(pTbDbc);
...@@ -220,9 +223,6 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { ...@@ -220,9 +223,6 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
metaReaderClear(&mr); metaReaderClear(&mr);
return -1; return -1;
} else {
pReq->uid = tGenIdPI64();
pReq->ctime = taosGetTimestampMs();
} }
metaReaderClear(&mr); metaReaderClear(&mr);
...@@ -256,9 +256,9 @@ _err: ...@@ -256,9 +256,9 @@ _err:
} }
int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) { int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
TBC * pTbDbc = NULL; TBC *pTbDbc = NULL;
TBC * pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TBC * pNameIdxc = NULL; TBC *pNameIdxc = NULL;
const void *pData; const void *pData;
int nData; int nData;
tb_uid_t uid; tb_uid_t uid;
...@@ -377,14 +377,14 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi ...@@ -377,14 +377,14 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
} }
static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
void * pVal = NULL; void *pVal = NULL;
int nVal = 0; int nVal = 0;
const void * pData = NULL; const void *pData = NULL;
int nData = 0; int nData = 0;
int ret = 0; int ret = 0;
tb_uid_t uid; tb_uid_t uid;
int64_t oversion; int64_t oversion;
SSchema * pColumn = NULL; SSchema *pColumn = NULL;
SMetaEntry entry = {0}; SMetaEntry entry = {0};
SSchemaWrapper *pSchema; SSchemaWrapper *pSchema;
int c; int c;
...@@ -420,7 +420,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl ...@@ -420,7 +420,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
// get table entry // get table entry
SDecoder dc = {0}; SDecoder dc = {0};
tDecoderInit(&dc, pData, nData); entry.pBuf = taosMemoryMalloc(nData);
memcpy(entry.pBuf, pData, nData);
tDecoderInit(&dc, entry.pBuf, nData);
ret = metaDecodeEntry(&dc, &entry); ret = metaDecodeEntry(&dc, &entry);
ASSERT(ret == 0); ASSERT(ret == 0);
...@@ -530,7 +532,7 @@ _err: ...@@ -530,7 +532,7 @@ _err:
static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
SMetaEntry ctbEntry = {0}; SMetaEntry ctbEntry = {0};
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
void * pVal = NULL; void *pVal = NULL;
int nVal = 0; int nVal = 0;
int ret; int ret;
int c; int c;
...@@ -561,7 +563,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ...@@ -561,7 +563,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
oversion = *(int64_t *)pData; oversion = *(int64_t *)pData;
// search table.db // search table.db
TBC * pTbDbc = NULL; TBC *pTbDbc = NULL;
SDecoder dc1 = {0}; SDecoder dc1 = {0};
SDecoder dc2 = {0}; SDecoder dc2 = {0};
...@@ -585,7 +587,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ...@@ -585,7 +587,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
metaDecodeEntry(&dc2, &stbEntry); metaDecodeEntry(&dc2, &stbEntry);
SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
SSchema * pColumn = NULL; SSchema *pColumn = NULL;
int32_t iCol = 0; int32_t iCol = 0;
for (;;) { for (;;) {
pColumn = NULL; pColumn = NULL;
...@@ -681,8 +683,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { ...@@ -681,8 +683,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) {
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
STbDbKey tbDbKey; STbDbKey tbDbKey;
void * pKey = NULL; void *pKey = NULL;
void * pVal = NULL; void *pVal = NULL;
int kLen = 0; int kLen = 0;
int vLen = 0; int vLen = 0;
SEncoder coder = {0}; SEncoder coder = {0};
...@@ -797,14 +799,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { ...@@ -797,14 +799,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
} }
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
void * pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
STbDbKey tbDbKey = {0}; STbDbKey tbDbKey = {0};
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
STagIdxKey * pTagIdxKey = NULL; STagIdxKey *pTagIdxKey = NULL;
int32_t nTagIdxKey; int32_t nTagIdxKey;
const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0];
const void * pTagData = NULL; // const void *pTagData = NULL; //
SDecoder dc = {0}; SDecoder dc = {0};
// get super table // get super table
...@@ -846,7 +848,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { ...@@ -846,7 +848,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
SEncoder coder = {0}; SEncoder coder = {0};
void * pVal = NULL; void *pVal = NULL;
int vLen = 0; int vLen = 0;
int rcode = 0; int rcode = 0;
SSkmDbKey skmDbKey = {0}; SSkmDbKey skmDbKey = {0};
......
...@@ -217,7 +217,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p ...@@ -217,7 +217,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p
if (tDecodeIsEnd(&dc)) break; if (tDecodeIsEnd(&dc)) break;
// decode row // decode row
if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { if (tDecodeBinary(&dc, (uint8_t **)&tRow.pRow, &tRow.szRow) < 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
return -1; return -1;
} }
...@@ -273,7 +273,7 @@ static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pR ...@@ -273,7 +273,7 @@ static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pR
static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) { static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) {
if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1; if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1;
if (tDecodeBinary(pDecoder, (const uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1; if (tDecodeBinary(pDecoder, (uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1;
return 0; return 0;
} }
......
...@@ -85,7 +85,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro ...@@ -85,7 +85,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro
return 0; return 0;
} }
int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) { int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
ASSERT(pMsg != NULL); ASSERT(pMsg != NULL);
// STsdbMeta * pMeta = pTsdb->tsdbMeta; // STsdbMeta * pMeta = pTsdb->tsdbMeta;
SSubmitMsgIter msgIter = {0}; SSubmitMsgIter msgIter = {0};
...@@ -150,7 +150,6 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) { ...@@ -150,7 +150,6 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) {
return -1; return -1;
} }
} }
} }
if (terrno != TSDB_CODE_SUCCESS) return -1; if (terrno != TSDB_CODE_SUCCESS) return -1;
......
...@@ -24,26 +24,62 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in ...@@ -24,26 +24,62 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in
static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
#if 0 SDecoder dc = {0};
SRpcMsg *pMsg;
SRpcMsg *pRpc;
*version = pVnode->state.processed; switch (pMsg->msgType) {
for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { case TDMT_VND_CREATE_TABLE: {
pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); int64_t ctime = taosGetTimestampMs();
pRpc = pMsg; int32_t nReqs;
// set request version tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead));
if (walWrite(pVnode->pWal, pVnode->state.processed++, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) { tStartDecode(&dc);
vError("vnode:%d write wal error since %s", TD_VID(pVnode), terrstr());
return -1; tDecodeI32v(&dc, &nReqs);
for (int32_t iReq = 0; iReq < nReqs; iReq++) {
tb_uid_t uid = tGenIdPI64();
tStartDecode(&dc);
tDecodeI32v(&dc, NULL);
*(int64_t *)(dc.data + dc.pos) = uid;
*(int64_t *)(dc.data + dc.pos + 8) = ctime;
tEndDecode(&dc);
}
tEndDecode(&dc);
tDecoderClear(&dc);
} break;
case TDMT_VND_SUBMIT: {
SSubmitMsgIter msgIter = {0};
SSubmitReq *pSubmitReq = (SSubmitReq *)pMsg->pCont;
SSubmitBlk *pBlock = NULL;
int64_t ctime = taosGetTimestampMs();
tInitSubmitMsgIter(pSubmitReq, &msgIter);
for (;;) {
tGetSubmitMsgNext(&msgIter, &pBlock);
if (pBlock == NULL) break;
if (msgIter.schemaLen > 0) {
tDecoderInit(&dc, pBlock->data, msgIter.schemaLen);
tStartDecode(&dc);
tDecodeI32v(&dc, NULL);
*(int64_t *)(dc.data + dc.pos) = tGenIdPI64();
*(int64_t *)(dc.data + dc.pos + 8) = ctime;
tEndDecode(&dc);
tDecoderClear(&dc);
} }
} }
walFsync(pVnode->pWal, false); } break;
default:
break;
}
#endif
return 0; return 0;
} }
...@@ -675,7 +711,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in ...@@ -675,7 +711,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
goto _exit; goto _exit;
} }
for (int i = 0;;) { for (;;) {
tGetSubmitMsgNext(&msgIter, &pBlock); tGetSubmitMsgNext(&msgIter, &pBlock);
if (pBlock == NULL) break; if (pBlock == NULL) break;
......
...@@ -58,6 +58,17 @@ enum { ...@@ -58,6 +58,17 @@ enum {
CTG_ACT_MAX CTG_ACT_MAX
}; };
typedef enum {
CTG_TASK_GET_QNODE = 0,
CTG_TASK_GET_DB_VGROUP,
CTG_TASK_GET_DB_CFG,
CTG_TASK_GET_TB_META,
CTG_TASK_GET_TB_HASH,
CTG_TASK_GET_INDEX,
CTG_TASK_GET_UDF,
CTG_TASK_GET_USER,
} CTG_TASK_TYPE;
typedef struct SCtgDebug { typedef struct SCtgDebug {
bool lockEnable; bool lockEnable;
bool cacheEnable; bool cacheEnable;
...@@ -66,6 +77,43 @@ typedef struct SCtgDebug { ...@@ -66,6 +77,43 @@ typedef struct SCtgDebug {
uint32_t showCachePeriodSec; uint32_t showCachePeriodSec;
} SCtgDebug; } SCtgDebug;
typedef struct SCtgTbCacheInfo {
bool inCache;
uint64_t dbId;
uint64_t suid;
int32_t tbType;
} SCtgTbCacheInfo;
typedef struct SCtgTbMetaCtx {
SCtgTbCacheInfo tbInfo;
SName* pName;
int32_t flag;
} SCtgTbMetaCtx;
typedef struct SCtgDbVgCtx {
char dbFName[TSDB_DB_FNAME_LEN];
} SCtgDbVgCtx;
typedef struct SCtgDbCfgCtx {
char dbFName[TSDB_DB_FNAME_LEN];
} SCtgDbCfgCtx;
typedef struct SCtgTbHashCtx {
char dbFName[TSDB_DB_FNAME_LEN];
SName* pName;
} SCtgTbHashCtx;
typedef struct SCtgIndexCtx {
char indexFName[TSDB_INDEX_FNAME_LEN];
} SCtgIndexCtx;
typedef struct SCtgUdfCtx {
char udfName[TSDB_FUNC_NAME_LEN];
} SCtgUdfCtx;
typedef struct SCtgUserCtx {
SUserAuthInfo user;
} SCtgUserCtx;
typedef struct SCtgTbMetaCache { typedef struct SCtgTbMetaCache {
SRWLatch stbLock; SRWLatch stbLock;
...@@ -113,6 +161,55 @@ typedef struct SCatalog { ...@@ -113,6 +161,55 @@ typedef struct SCatalog {
SCtgRentMgmt stbRent; SCtgRentMgmt stbRent;
} SCatalog; } SCatalog;
typedef struct SCtgJob {
int64_t refId;
SArray* pTasks;
int32_t taskDone;
SMetaData jobRes;
int32_t rspCode;
uint64_t queryId;
SCatalog* pCtg;
void* pTrans;
const SEpSet* pMgmtEps;
void* userParam;
catalogCallback userFp;
int32_t tbMetaNum;
int32_t tbHashNum;
int32_t dbVgNum;
int32_t udfNum;
int32_t qnodeNum;
int32_t dbCfgNum;
int32_t indexNum;
int32_t userNum;
} SCtgJob;
typedef struct SCtgMsgCtx {
int32_t reqType;
void* lastOut;
void* out;
char* target;
} SCtgMsgCtx;
typedef struct SCtgTask {
CTG_TASK_TYPE type;
int32_t taskId;
SCtgJob *pJob;
void* taskCtx;
SCtgMsgCtx msgCtx;
void* res;
} SCtgTask;
typedef int32_t (*ctgLanchTaskFp)(SCtgTask*);
typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTask*, int32_t, const SDataBuf *, int32_t);
typedef int32_t (*ctgDumpTaskResFp)(SCtgTask*);
typedef struct SCtgAsyncFps {
ctgLanchTaskFp launchFp;
ctgHandleTaskMsgRspFp handleRspFp;
ctgDumpTaskResFp dumpResFp;
} SCtgAsyncFps;
typedef struct SCtgApiStat { typedef struct SCtgApiStat {
#ifdef WINDOWS #ifdef WINDOWS
...@@ -214,6 +311,7 @@ typedef struct SCtgQueue { ...@@ -214,6 +311,7 @@ typedef struct SCtgQueue {
typedef struct SCatalogMgmt { typedef struct SCatalogMgmt {
bool exit; bool exit;
int32_t jobPool;
SRWLatch lock; SRWLatch lock;
SCtgQueue queue; SCtgQueue queue;
TdThread updateThread; TdThread updateThread;
...@@ -327,10 +425,80 @@ typedef struct SCtgAction { ...@@ -327,10 +425,80 @@ typedef struct SCtgAction {
#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) #define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0)
#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0)
#define CTG_PARAMS SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps
extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); #define CTG_PARAMS_LIST() pCtg, pTrans, pMgmtEps
extern void ctgdShowClusterCache(SCatalog* pCtg);
extern int32_t ctgdShowCacheInfo(void); void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p);
void ctgdShowClusterCache(SCatalog* pCtg);
int32_t ctgdShowCacheInfo(void);
int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq);
int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgActUpdateVg(SCtgMetaAction *action);
int32_t ctgActUpdateTb(SCtgMetaAction *action);
int32_t ctgActRemoveDB(SCtgMetaAction *action);
int32_t ctgActRemoveStb(SCtgMetaAction *action);
int32_t ctgActRemoveTb(SCtgMetaAction *action);
int32_t ctgActUpdateUser(SCtgMetaAction *action);
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache);
void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache);
void ctgReleaseVgInfo(SCtgDBCache *dbCache);
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache);
int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist);
int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgReadTbSverFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tbType, uint64_t *suid, char *stbName);
int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass);
int32_t ctgPutRmDBToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId);
int32_t ctgPutRmStbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq);
int32_t ctgPutRmTbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq);
int32_t ctgPutUpdateVgToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq);
int32_t ctgPutUpdateTbToQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq);
int32_t ctgPutUpdateUserToQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq);
int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type);
int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size);
int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size);
int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq);
int32_t ctgStartUpdateThread();
int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask);
int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target);
int32_t ctgGetDBVgInfoFromMnode(CTG_PARAMS, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask);
int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask);
int32_t ctgGetDBCfgFromMnode(CTG_PARAMS, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask);
int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *out, SCtgTask* pTask);
int32_t ctgGetUdfInfoFromMnode(CTG_PARAMS, const char *funcName, SFuncInfo *out, SCtgTask* pTask);
int32_t ctgGetUserDbAuthFromMnode(CTG_PARAMS, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask);
int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask);
int32_t ctgGetTbMetaFromMnode(CTG_PARAMS, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask);
int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask);
int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param);
int32_t ctgLaunchJob(SCtgJob *pJob);
int32_t ctgMakeAsyncRes(SCtgJob *pJob);
int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst);
int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput);
int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList);
void ctgFreeJob(void* job);
void ctgFreeHandle(SCatalog* pCtg);
void ctgFreeVgInfo(SDBVgInfo *vgInfo);
int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup);
void ctgResetTbMetaTask(SCtgTask* pTask);
void ctgFreeDbCache(SCtgDBCache *dbCache);
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2);
int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2);
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2);
int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2);
void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput);
int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target);
extern SCatalogMgmt gCtgMgmt;
extern SCtgDebug gCTGDebug;
extern SCtgAsyncFps gCtgAsyncFps[];
#ifdef __cplusplus #ifdef __cplusplus
} }
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_CATALOG_REMOTE_H_
#define _TD_CATALOG_REMOTE_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SCtgTaskCallbackParam {
uint64_t queryId;
int64_t refId;
uint64_t taskId;
int32_t reqType;
} SCtgTaskCallbackParam;
#ifdef __cplusplus
}
#endif
#endif /*_TD_CATALOG_REMOTE_H_*/
...@@ -13,1773 +13,52 @@ ...@@ -13,1773 +13,52 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "catalogInt.h" #include "trpc.h"
#include "query.h" #include "query.h"
#include "systable.h" #include "tname.h"
#include "tname.h" #include "catalogInt.h"
#include "trpc.h" #include "systable.h"
#include "tref.h"
int32_t ctgActUpdateVg(SCtgMetaAction *action);
int32_t ctgActUpdateTbl(SCtgMetaAction *action);
int32_t ctgActRemoveDB(SCtgMetaAction *action);
int32_t ctgActRemoveStb(SCtgMetaAction *action);
int32_t ctgActRemoveTbl(SCtgMetaAction *action);
int32_t ctgActUpdateUser(SCtgMetaAction *action);
extern SCtgDebug gCTGDebug;
SCatalogMgmt gCtgMgmt = {0};
SCtgAction gCtgAction[CTG_ACT_MAX] = {
{CTG_ACT_UPDATE_VG, "update vgInfo", ctgActUpdateVg}, {CTG_ACT_UPDATE_TBL, "update tbMeta", ctgActUpdateTbl},
{CTG_ACT_REMOVE_DB, "remove DB", ctgActRemoveDB}, {CTG_ACT_REMOVE_STB, "remove stbMeta", ctgActRemoveStb},
{CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl}, {CTG_ACT_UPDATE_USER, "update user", ctgActUpdateUser}};
void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
if (NULL == mgmt->slots) {
return;
}
for (int32_t i = 0; i < mgmt->slotNum; ++i) {
SCtgRentSlot *slot = &mgmt->slots[i];
if (slot->meta) {
taosArrayDestroy(slot->meta);
slot->meta = NULL;
}
}
taosMemoryFreeClear(mgmt->slots);
}
void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) {
CTG_LOCK(CTG_WRITE, &cache->stbLock);
if (cache->stbCache) {
int32_t stblNum = taosHashGetSize(cache->stbCache);
taosHashCleanup(cache->stbCache);
cache->stbCache = NULL;
CTG_CACHE_STAT_SUB(stblNum, stblNum);
}
CTG_UNLOCK(CTG_WRITE, &cache->stbLock);
CTG_LOCK(CTG_WRITE, &cache->metaLock);
if (cache->metaCache) {
int32_t tblNum = taosHashGetSize(cache->metaCache);
taosHashCleanup(cache->metaCache);
cache->metaCache = NULL;
CTG_CACHE_STAT_SUB(tblNum, tblNum);
}
CTG_UNLOCK(CTG_WRITE, &cache->metaLock);
}
void ctgFreeVgInfo(SDBVgInfo *vgInfo) {
if (NULL == vgInfo) {
return;
}
if (vgInfo->vgHash) {
taosHashCleanup(vgInfo->vgHash);
vgInfo->vgHash = NULL;
}
taosMemoryFreeClear(vgInfo);
}
void ctgFreeDbCache(SCtgDBCache *dbCache) {
if (NULL == dbCache) {
return;
}
CTG_LOCK(CTG_WRITE, &dbCache->vgLock);
ctgFreeVgInfo(dbCache->vgInfo);
CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock);
ctgFreeTableMetaCache(&dbCache->tbCache);
}
void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) {
taosHashCleanup(userCache->createdDbs);
taosHashCleanup(userCache->readDbs);
taosHashCleanup(userCache->writeDbs);
}
void ctgFreeHandle(SCatalog *pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
if (pCtg->dbCache) {
int32_t dbNum = taosHashGetSize(pCtg->dbCache);
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
while (pIter) {
SCtgDBCache *dbCache = pIter;
atomic_store_8(&dbCache->deleted, 1);
ctgFreeDbCache(dbCache);
pIter = taosHashIterate(pCtg->dbCache, pIter);
}
taosHashCleanup(pCtg->dbCache);
CTG_CACHE_STAT_SUB(dbNum, dbNum);
}
if (pCtg->userCache) {
int32_t userNum = taosHashGetSize(pCtg->userCache);
void *pIter = taosHashIterate(pCtg->userCache, NULL);
while (pIter) {
SCtgUserAuth *userCache = pIter;
ctgFreeSCtgUserAuth(userCache);
pIter = taosHashIterate(pCtg->userCache, pIter);
}
taosHashCleanup(pCtg->userCache);
CTG_CACHE_STAT_SUB(userNum, userNum);
}
taosMemoryFree(pCtg);
}
void ctgWaitAction(SCtgMetaAction *action) {
while (true) {
tsem_wait(&gCtgMgmt.queue.rspSem);
if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) {
tsem_post(&gCtgMgmt.queue.rspSem);
break;
}
if (gCtgMgmt.queue.seqDone >= action->seqId) {
break;
}
tsem_post(&gCtgMgmt.queue.rspSem);
sched_yield();
}
}
void ctgPopAction(SCtgMetaAction **action) {
SCtgQNode *orig = gCtgMgmt.queue.head;
SCtgQNode *node = gCtgMgmt.queue.head->next;
gCtgMgmt.queue.head = gCtgMgmt.queue.head->next;
CTG_QUEUE_SUB();
taosMemoryFreeClear(orig);
*action = &node->action;
}
int32_t ctgPushAction(SCatalog *pCtg, SCtgMetaAction *action) {
SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode));
if (NULL == node) {
qError("calloc %d failed", (int32_t)sizeof(SCtgQNode));
CTG_RET(TSDB_CODE_CTG_MEM_ERROR);
}
action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1);
node->action = *action;
CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
gCtgMgmt.queue.tail->next = node;
gCtgMgmt.queue.tail = node;
CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
CTG_QUEUE_ADD();
CTG_RUNTIME_STAT_ADD(qNum, 1);
tsem_post(&gCtgMgmt.queue.reqSem);
ctgDebug("action [%s] added into queue", gCtgAction[action->act].name);
if (action->syncReq) {
ctgWaitAction(action);
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgPushRmDBMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) {
int32_t code = 0;
SCtgMetaAction action = {.act = CTG_ACT_REMOVE_DB};
SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
char *p = strchr(dbFName, '.');
if (p && CTG_IS_SYS_DBNAME(p + 1)) {
dbFName = p + 1;
}
msg->pCtg = pCtg;
strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
msg->dbId = dbId;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(action.data);
CTG_RET(code);
}
int32_t ctgPushRmStbMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid,
bool syncReq) {
int32_t code = 0;
SCtgMetaAction action = {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq};
SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
msg->pCtg = pCtg;
strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
strncpy(msg->stbName, stbName, sizeof(msg->stbName));
msg->dbId = dbId;
msg->suid = suid;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(action.data);
CTG_RET(code);
}
int32_t ctgPushRmTblMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action = {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq};
SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
msg->pCtg = pCtg;
strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
strncpy(msg->tbName, tbName, sizeof(msg->tbName));
msg->dbId = dbId;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(action.data);
CTG_RET(code);
}
int32_t ctgPushUpdateVgMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDBVgInfo *dbInfo, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action = {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq};
SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg));
ctgFreeVgInfo(dbInfo);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
char *p = strchr(dbFName, '.');
if (p && CTG_IS_SYS_DBNAME(p + 1)) {
dbFName = p + 1;
}
strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
msg->pCtg = pCtg;
msg->dbId = dbId;
msg->dbInfo = dbInfo;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
ctgFreeVgInfo(dbInfo);
taosMemoryFreeClear(action.data);
CTG_RET(code);
}
int32_t ctgPushUpdateTblMsgInQueue(SCatalog *pCtg, STableMetaOutput *output, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action = {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq};
SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
char *p = strchr(output->dbFName, '.');
if (p && CTG_IS_SYS_DBNAME(p + 1)) {
memmove(output->dbFName, p + 1, strlen(p + 1));
}
msg->pCtg = pCtg;
msg->output = output;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgPushUpdateUserMsgInQueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action = {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq};
SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
msg->pCtg = pCtg;
msg->userAuth = *pAuth;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
tFreeSGetUserAuthRsp(pAuth);
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) {
CTG_LOCK(CTG_READ, &dbCache->vgLock);
if (dbCache->deleted) {
CTG_UNLOCK(CTG_READ, &dbCache->vgLock);
ctgDebug("db is dropping, dbId:%" PRIx64, dbCache->dbId);
*inCache = false;
return TSDB_CODE_SUCCESS;
}
if (NULL == dbCache->vgInfo) {
CTG_UNLOCK(CTG_READ, &dbCache->vgLock);
*inCache = false;
ctgDebug("db vgInfo is empty, dbId:%" PRIx64, dbCache->dbId);
return TSDB_CODE_SUCCESS;
}
*inCache = true;
return TSDB_CODE_SUCCESS;
}
int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) {
CTG_LOCK(CTG_WRITE, &dbCache->vgLock);
if (dbCache->deleted) {
ctgDebug("db is dropping, dbId:%" PRIx64, dbCache->dbId);
CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock);
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
return TSDB_CODE_SUCCESS;
}
void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { taosHashRelease(pCtg->dbCache, dbCache); }
void ctgReleaseVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); }
void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); }
int32_t ctgAcquireDBCacheImpl(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) {
char *p = strchr(dbFName, '.');
if (p && CTG_IS_SYS_DBNAME(p + 1)) {
dbFName = p + 1;
}
SCtgDBCache *dbCache = NULL;
if (acquire) {
dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName));
} else {
dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName));
}
if (NULL == dbCache) {
*pCache = NULL;
ctgDebug("db not in cache, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
}
if (dbCache->deleted) {
if (acquire) {
ctgReleaseDBCache(pCtg, dbCache);
}
*pCache = NULL;
ctgDebug("db is removing from cache, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
}
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
int32_t ctgAcquireDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) {
CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true));
}
int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) {
CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false));
}
int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) {
SCtgDBCache *dbCache = NULL;
if (NULL == pCtg->dbCache) {
ctgDebug("empty db cache, dbFName:%s", dbFName);
goto _return;
}
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
ctgDebug("db %s not in cache", dbFName);
goto _return;
}
ctgAcquireVgInfo(pCtg, dbCache, inCache);
if (!(*inCache)) {
ctgDebug("vgInfo of db %s not in cache", dbFName);
goto _return;
}
*pCache = dbCache;
*inCache = true;
CTG_CACHE_STAT_ADD(vgHitNum, 1);
ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
_return:
if (dbCache) {
ctgReleaseDBCache(pCtg, dbCache);
}
*pCache = NULL;
*inCache = false;
CTG_CACHE_STAT_ADD(vgMissNum, 1);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetQnodeListFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SArray *out) {
char *msg = NULL;
int32_t msgLen = 0;
ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pMgmtEps->inUse);
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)](NULL, &msg, 0, &msgLen);
if (code) {
ctgError("Build qnode list msg failed, error:%s", tstrerror(code));
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_QNODE_LIST,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("error rsp for qnode list, error:%s", tstrerror(rpcRsp.code));
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)](out, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process qnode list rsp failed, error:%s", tstrerror(rpcRsp.code));
CTG_ERR_RET(code);
}
ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetDBVgInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SBuildUseDBInput *input,
SUseDbOutput *out) {
char *msg = NULL;
int32_t msgLen = 0;
ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db);
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)](input, &msg, 0, &msgLen);
if (code) {
ctgError("Build use db msg failed, code:%x, db:%s", code, input->db);
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_USE_DB,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("error rsp for use db, error:%s, db:%s", tstrerror(rpcRsp.code), input->db);
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)](out, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process use db rsp failed, code:%x, db:%s", code, input->db);
CTG_ERR_RET(code);
}
ctgDebug("Got db vgInfo from mnode, dbFName:%s", input->db);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetDBCfgFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SDbCfgInfo *out) {
char *msg = NULL;
int32_t msgLen = 0;
ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName);
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)]((void *)dbFName, &msg, 0, &msgLen);
if (code) {
ctgError("Build get db cfg msg failed, code:%x, db:%s", code, dbFName);
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_GET_DB_CFG,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("error rsp for get db cfg, error:%s, db:%s", tstrerror(rpcRsp.code), dbFName);
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)](out, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process get db cfg rsp failed, code:%x, db:%s", code, dbFName);
CTG_ERR_RET(code);
}
ctgDebug("Got db cfg from mnode, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetIndexInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *indexName,
SIndexInfo *out) {
char *msg = NULL;
int32_t msgLen = 0;
ctgDebug("try to get index from mnode, indexName:%s", indexName);
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)]((void *)indexName, &msg, 0, &msgLen);
if (code) {
ctgError("Build get index msg failed, code:%x, db:%s", code, indexName);
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_GET_INDEX,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("error rsp for get index, error:%s, indexName:%s", tstrerror(rpcRsp.code), indexName);
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)](out, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process get index rsp failed, code:%x, indexName:%s", code, indexName);
CTG_ERR_RET(code);
}
ctgDebug("Got index from mnode, indexName:%s", indexName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetUdfInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *funcName,
SFuncInfo **out) {
char *msg = NULL;
int32_t msgLen = 0;
ctgDebug("try to get udf info from mnode, funcName:%s", funcName);
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]((void *)funcName, &msg, 0, &msgLen);
if (code) {
ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName);
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_RETRIEVE_FUNC,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) {
ctgDebug("funcName %s not exist in mnode", funcName);
taosMemoryFreeClear(*out);
CTG_RET(TSDB_CODE_SUCCESS);
}
ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName);
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)](*out, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process get udf rsp failed, code:%x, funcName:%s", code, funcName);
CTG_ERR_RET(code);
}
ctgDebug("Got udf from mnode, funcName:%s", funcName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetUserDbAuthFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user,
SGetUserAuthRsp *authRsp) {
char *msg = NULL;
int32_t msgLen = 0;
ctgDebug("try to get user auth from mnode, user:%s", user);
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)]((void *)user, &msg, 0, &msgLen);
if (code) {
ctgError("Build get user auth msg failed, code:%x, db:%s", code, user);
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_GET_USER_AUTH,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user);
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)](authRsp, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process get user auth rsp failed, code:%x, user:%s", code, user);
CTG_ERR_RET(code);
}
ctgDebug("Got user auth from mnode, user:%s", user);
return TSDB_CODE_SUCCESS;
}
int32_t ctgIsTableMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32_t *exist) {
if (NULL == pCtg->dbCache) {
*exist = 0;
ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName);
return TSDB_CODE_SUCCESS;
}
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
size_t sz = 0;
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName));
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == tbMeta) {
ctgReleaseDBCache(pCtg, dbCache);
*exist = 0;
ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName);
return TSDB_CODE_SUCCESS;
}
*exist = 1;
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableMetaFromCache(SCatalog *pCtg, const SName *pTableName, STableMeta **pTableMeta, bool *inCache,
int32_t flag, uint64_t *dbId) {
if (NULL == pCtg->dbCache) {
ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname);
goto _return;
}
char dbFName[TSDB_DB_FNAME_LEN] = {0};
if (CTG_FLAG_IS_SYS_DB(flag)) {
strcpy(dbFName, pTableName->dbname);
} else {
tNameGetFullDbName(pTableName, dbFName);
}
*pTableMeta = NULL;
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
ctgDebug("db %s not in cache", pTableName->tname);
goto _return;
}
int32_t sz = 0;
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname),
(void **)pTableMeta, &sz);
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == *pTableMeta) {
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname);
goto _return;
}
if (dbId) {
*dbId = dbCache->dbId;
}
STableMeta *tbMeta = *pTableMeta;
if (tbMeta->tableType != TSDB_CHILD_TABLE) {
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
*inCache = true;
CTG_CACHE_STAT_ADD(tblHitNum, 1);
return TSDB_CODE_SUCCESS;
}
ctgDebug("Got subtable meta from cache, type:%d, dbFName:%s, tbName:%s, suid:%" PRIx64, tbMeta->tableType, dbFName,
pTableName->tname, tbMeta->suid);
CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock);
STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid));
if (NULL == stbMeta || NULL == *stbMeta) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgError("stb not in stbCache, suid:%" PRIx64, tbMeta->suid);
taosMemoryFreeClear(*pTableMeta);
goto _return;
}
if ((*stbMeta)->suid != tbMeta->suid) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
taosMemoryFreeClear(*pTableMeta);
ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, tbMeta->suid,
(*stbMeta)->suid);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t metaSize = CTG_META_SIZE(*stbMeta);
*pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize);
if (NULL == *pTableMeta) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgError("realloc size[%d] failed", metaSize);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta));
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
*inCache = true;
CTG_CACHE_STAT_ADD(tblHitNum, 1);
ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname);
return TSDB_CODE_SUCCESS;
_return:
*inCache = false;
CTG_CACHE_STAT_ADD(tblMissNum, 1);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableTypeFromCache(SCatalog *pCtg, const char *dbFName, const char *tableName, int32_t *tbType) {
if (NULL == pCtg->dbCache) {
ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName);
return TSDB_CODE_SUCCESS;
}
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName));
if (NULL == pTableMeta) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName);
ctgReleaseDBCache(pCtg, dbCache);
return TSDB_CODE_SUCCESS;
}
*tbType = atomic_load_8(&pTableMeta->tableType);
taosHashRelease(dbCache->tbCache.metaCache, pTableMeta);
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType);
return TSDB_CODE_SUCCESS;
}
int32_t ctgChkAuthFromCache(SCatalog *pCtg, const char *user, const char *dbFName, AUTH_TYPE type, bool *inCache,
bool *pass) {
if (NULL == pCtg->userCache) {
ctgDebug("empty user auth cache, user:%s", user);
goto _return;
}
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user));
if (NULL == pUser) {
ctgDebug("user not in cache, user:%s", user);
goto _return;
}
*inCache = true;
ctgDebug("Got user from cache, user:%s", user);
CTG_CACHE_STAT_ADD(userHitNum, 1);
if (pUser->superUser) {
*pass = true;
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &pUser->lock);
if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) {
*pass = true;
CTG_UNLOCK(CTG_READ, &pUser->lock);
return TSDB_CODE_SUCCESS;
}
if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) {
*pass = true;
}
if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) {
*pass = true;
}
CTG_UNLOCK(CTG_READ, &pUser->lock);
return TSDB_CODE_SUCCESS;
_return:
*inCache = false;
CTG_CACHE_STAT_ADD(userMissNum, 1);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableMetaFromMnodeImpl(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, char *dbFName, char *tbName,
STableMetaOutput *output) {
SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName};
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName);
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)](&bInput, &msg, 0, &msgLen);
if (code) {
ctgError("Build mnode stablemeta msg failed, code:%x", code);
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_TABLE_META,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) {
SET_META_TYPE_NULL(output->metaType);
ctgDebug("stablemeta not exist in mnode, dbFName:%s, tbName:%s", dbFName, tbName);
return TSDB_CODE_SUCCESS;
}
ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName,
tbName);
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName);
CTG_ERR_RET(code);
}
ctgDebug("Got table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableMetaFromMnode(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName,
STableMetaOutput *output) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output);
}
int32_t ctgGetTableMetaFromVnodeImpl(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName,
SVgroupInfo *vgroupInfo, STableMetaOutput *output) {
if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo ||
NULL == output) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
ctgDebug("try to get table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName));
SBuildTableMetaInput bInput = {
.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)};
char *msg = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen);
if (code) {
ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName,
tNameGetTableName(pTableName));
CTG_ERR_RET(code);
}
SRpcMsg rpcMsg = {
.msgType = TDMT_VND_TABLE_META,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) {
SET_META_TYPE_NULL(output->metaType);
ctgDebug("tablemeta not exist in vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName));
return TSDB_CODE_SUCCESS;
}
ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName,
tNameGetTableName(pTableName));
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName,
tNameGetTableName(pTableName));
CTG_ERR_RET(code);
}
ctgDebug("Got table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableMetaFromVnode(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName,
SVgroupInfo *vgroupInfo, STableMetaOutput *output) {
int32_t code = 0;
int32_t retryNum = 0;
while (retryNum < CTG_DEFAULT_MAX_RETRY_TIMES) {
code = ctgGetTableMetaFromVnodeImpl(pCtg, pTrans, pMgmtEps, pTableName, vgroupInfo, output);
if (code) {
if (TSDB_CODE_VND_HASH_MISMATCH == code) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
code = catalogRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName);
if (code != TSDB_CODE_SUCCESS) {
break;
}
++retryNum;
continue;
}
}
break;
}
CTG_RET(code);
}
int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) {
switch (hashMethod) {
default:
*fp = MurmurHash3_32;
break;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray **pList) {
SHashObj *vgroupHash = NULL;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
int32_t code = 0;
int32_t vgNum = taosHashGetSize(vgHash);
vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo));
if (NULL == vgList) {
ctgError("taosArrayInit failed, num:%d", vgNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
void *pIter = taosHashIterate(vgHash, NULL);
while (pIter) {
vgInfo = pIter;
if (NULL == taosArrayPush(vgList, vgInfo)) {
ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId);
taosHashCancelIterate(vgHash, pIter);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
pIter = taosHashIterate(vgHash, pIter);
vgInfo = NULL;
}
*pList = vgList;
ctgDebug("Got vgList from cache, vgNum:%d", vgNum);
return TSDB_CODE_SUCCESS;
_return:
if (vgList) {
taosArrayDestroy(vgList);
}
CTG_RET(code);
}
int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) {
int32_t code = 0;
int32_t vgNum = taosHashGetSize(dbInfo->vgHash);
char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
if (vgNum <= 0) {
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
}
tableNameHashFp fp = NULL;
SVgroupInfo *vgInfo = NULL;
CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp));
char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName));
void *pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) {
vgInfo = pIter;
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
taosHashCancelIterate(dbInfo->vgHash, pIter);
break;
}
pIter = taosHashIterate(dbInfo->vgHash, pIter);
vgInfo = NULL;
}
if (NULL == vgInfo) {
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db,
taosHashGetSize(dbInfo->vgHash));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
*pVgroup = *vgInfo;
CTG_RET(code);
}
int32_t ctgStbVersionSearchCompare(const void *key1, const void *key2) {
if (*(uint64_t *)key1 < ((SSTableMetaVersion *)key2)->suid) {
return -1;
} else if (*(uint64_t *)key1 > ((SSTableMetaVersion *)key2)->suid) {
return 1;
} else {
return 0;
}
}
int32_t ctgDbVgVersionSearchCompare(const void *key1, const void *key2) {
if (*(int64_t *)key1 < ((SDbVgVersion *)key2)->dbId) {
return -1;
} else if (*(int64_t *)key1 > ((SDbVgVersion *)key2)->dbId) {
return 1;
} else {
return 0;
}
}
int32_t ctgStbVersionSortCompare(const void *key1, const void *key2) {
if (((SSTableMetaVersion *)key1)->suid < ((SSTableMetaVersion *)key2)->suid) {
return -1;
} else if (((SSTableMetaVersion *)key1)->suid > ((SSTableMetaVersion *)key2)->suid) {
return 1;
} else {
return 0;
}
}
int32_t ctgDbVgVersionSortCompare(const void *key1, const void *key2) {
if (((SDbVgVersion *)key1)->dbId < ((SDbVgVersion *)key2)->dbId) {
return -1;
} else if (((SDbVgVersion *)key1)->dbId > ((SDbVgVersion *)key2)->dbId) {
return 1;
} else {
return 0;
}
}
int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) {
mgmt->slotRIdx = 0;
mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND;
mgmt->type = type;
size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum;
mgmt->slots = taosMemoryCalloc(1, msgSize);
if (NULL == mgmt->slots) {
qError("calloc %d failed", (int32_t)msgSize);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum);
return TSDB_CODE_SUCCESS;
}
int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size);
if (NULL == slot->meta) {
qError("taosArrayInit %d failed, id:%" PRIx64 ", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx,
mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
}
if (NULL == taosArrayPush(slot->meta, meta)) {
qError("taosArrayPush meta to rent failed, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
slot->needSort = true;
qDebug("add meta to rent, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare,
__compar_fn_t searchCompare) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
qError("empty meta slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (slot->needSort) {
qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type,
(int32_t)taosArrayGetSize(slot->meta));
taosArraySort(slot->meta, sortCompare);
slot->needSort = false;
qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta));
}
void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ);
if (NULL == orig) {
qError("meta not found in slot, id:%" PRIx64 ", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type,
(int32_t)taosArrayGetSize(slot->meta));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
memcpy(orig, meta, size);
qDebug("meta in rent updated, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
if (code) {
qWarn("meta in rent update failed, will try to add it, code:%x, id:%" PRIx64 ", slot idx:%d, type:%d", code, id,
widx, mgmt->type);
CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size));
}
CTG_RET(code);
}
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
qError("empty meta slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (slot->needSort) {
taosArraySort(slot->meta, sortCompare);
slot->needSort = false;
qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type);
}
int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ);
if (idx < 0) {
qError("meta not found in slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
taosArrayRemove(slot->meta, idx);
qDebug("meta in rent removed, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) {
int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1);
if (ridx >= mgmt->slotNum) {
ridx %= mgmt->slotNum;
atomic_store_16(&mgmt->slotRIdx, ridx);
}
SCtgRentSlot *slot = &mgmt->slots[ridx];
int32_t code = 0;
CTG_LOCK(CTG_READ, &slot->lock);
if (NULL == slot->meta) {
qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type);
*num = 0;
goto _return;
}
size_t metaNum = taosArrayGetSize(slot->meta);
if (metaNum <= 0) {
qDebug("no meta in slot:%d, type:%d", ridx, mgmt->type);
*num = 0;
goto _return;
}
size_t msize = metaNum * size;
*res = taosMemoryMalloc(msize);
if (NULL == *res) {
qError("malloc %d failed", (int32_t)msize);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
void *meta = taosArrayGet(slot->meta, 0);
memcpy(*res, meta, msize);
*num = (uint32_t)metaNum;
qDebug("Got %d meta from rent, type:%d", (int32_t)metaNum, mgmt->type);
_return:
CTG_UNLOCK(CTG_READ, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) {
while (true) {
int64_t msec = taosGetTimestampMs();
int64_t lsec = atomic_load_64(&mgmt->lastReadMsec);
if ((msec - lsec) < CTG_RENT_SLOT_SECOND * 1000) {
*res = NULL;
*num = 0;
qDebug("too short time period to get expired meta, type:%d", mgmt->type);
return TSDB_CODE_SUCCESS;
}
if (lsec != atomic_val_compare_exchange_64(&mgmt->lastReadMsec, lsec, msec)) {
continue;
}
break;
}
CTG_ERR_RET(ctgMetaRentGetImpl(mgmt, res, num, size));
return TSDB_CODE_SUCCESS;
}
int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
int32_t code = 0;
SCtgDBCache newDBCache = {0};
newDBCache.dbId = dbId;
newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == newDBCache.tbCache.metaCache) {
ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
if (NULL == newDBCache.tbCache.stbCache) {
ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache));
if (code) {
if (HASH_NODE_EXIST(code)) {
ctgDebug("db already in cache, dbFName:%s", dbFName);
goto _return;
}
ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
CTG_CACHE_STAT_ADD(dbNum, 1);
SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1};
strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
ctgDebug("db added to cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbId);
CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion)));
ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%" PRIx64, dbFName, vgVersion.vgVersion, dbId);
return TSDB_CODE_SUCCESS;
_return:
ctgFreeDbCache(&newDBCache);
CTG_RET(code);
}
void ctgRemoveStbRent(SCatalog *pCtg, SCtgTbMetaCache *cache) {
CTG_LOCK(CTG_WRITE, &cache->stbLock);
if (cache->stbCache) {
void *pIter = taosHashIterate(cache->stbCache, NULL);
while (pIter) {
uint64_t *suid = NULL;
suid = taosHashGetKey(pIter, NULL);
if (TSDB_CODE_SUCCESS ==
ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) {
ctgDebug("stb removed from rent, suid:%" PRIx64, *suid);
}
pIter = taosHashIterate(cache->stbCache, pIter);
}
}
CTG_UNLOCK(CTG_WRITE, &cache->stbLock);
}
int32_t ctgRemoveDB(SCatalog *pCtg, SCtgDBCache *dbCache, const char *dbFName) {
uint64_t dbId = dbCache->dbId;
ctgInfo("start to remove db from cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbCache->dbId);
atomic_store_8(&dbCache->deleted, 1);
ctgRemoveStbRent(pCtg, &dbCache->tbCache);
ctgFreeDbCache(dbCache);
CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare));
ctgDebug("db removed from rent, dbFName:%s, dbId:%" PRIx64, dbFName, dbCache->dbId);
if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) {
ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName);
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
CTG_CACHE_STAT_SUB(dbNum, 1);
ctgInfo("db removed from cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbId);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) {
int32_t code = 0;
SCtgDBCache *dbCache = NULL;
ctgGetDBCache(pCtg, dbFName, &dbCache);
if (dbCache) {
// TODO OPEN IT
#if 0
if (dbCache->dbId == dbId) {
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
#else
if (0 == dbId) {
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
if (dbId && (dbCache->dbId == 0)) {
dbCache->dbId = dbId;
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
if (dbCache->dbId == dbId) {
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
#endif
CTG_ERR_RET(ctgRemoveDB(pCtg, dbCache, dbFName));
}
CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId));
ctgGetDBCache(pCtg, dbFName, &dbCache);
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
int32_t ctgUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SDBVgInfo **pDbInfo) {
int32_t code = 0;
SDBVgInfo *dbInfo = *pDbInfo;
if (NULL == dbInfo->vgHash) {
return TSDB_CODE_SUCCESS;
}
if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) {
ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash,
dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
bool newAdded = false;
SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable};
SCtgDBCache *dbCache = NULL;
CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache));
if (NULL == dbCache) {
ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, dbFName, dbId);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SDBVgInfo *vgInfo = NULL;
CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache));
if (dbCache->vgInfo) {
if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) {
ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion,
dbCache->vgInfo->vgVersion);
ctgWReleaseVgInfo(dbCache);
return TSDB_CODE_SUCCESS;
}
if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) {
ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion,
dbInfo->numOfTable);
ctgWReleaseVgInfo(dbCache);
return TSDB_CODE_SUCCESS;
}
ctgFreeVgInfo(dbCache->vgInfo);
}
dbCache->vgInfo = dbInfo;
*pDbInfo = NULL;
ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId);
ctgWReleaseVgInfo(dbCache);
dbCache = NULL;
strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion),
ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare));
CTG_RET(code);
}
int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName,
STableMeta *meta, int32_t metaSize) {
SCtgTbMetaCache *tbCache = &dbCache->tbCache;
CTG_LOCK(CTG_READ, &tbCache->metaLock);
if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) {
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgError("db is dropping, dbId:%" PRIx64, dbCache->dbId);
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
int8_t origType = 0;
uint64_t origSuid = 0;
bool isStb = meta->tableType == TSDB_SUPER_TABLE;
STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName));
if (orig) {
origType = orig->tableType;
if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion &&
orig->tversion >= meta->tversion) {
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
return TSDB_CODE_SUCCESS;
}
if (origType == TSDB_SUPER_TABLE) {
if ((!isStb) || orig->suid != meta->suid) {
CTG_LOCK(CTG_WRITE, &tbCache->stbLock);
if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) {
ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%" PRIx64, dbFName, tbName, orig->suid);
} else {
CTG_CACHE_STAT_SUB(stblNum, 1);
}
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%" PRIx64, dbFName, tbName, orig->suid);
ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare);
}
origSuid = orig->suid;
}
}
if (isStb) {
CTG_LOCK(CTG_WRITE, &tbCache->stbLock);
}
if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) {
if (isStb) {
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
}
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
if (NULL == orig) {
CTG_CACHE_STAT_ADD(tblNum, 1);
}
ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64, dbFName, tbName, meta->tableType,
meta->suid);
ctgdShowTableMeta(pCtg, tbName, meta);
if (!isStb) {
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
return TSDB_CODE_SUCCESS;
}
STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName));
if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) {
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgError("taosHashPut stable to stable cache failed, suid:%" PRIx64, meta->suid);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
CTG_CACHE_STAT_ADD(stblNum, 1);
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); SCatalogMgmt gCtgMgmt = {0};
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64 ",ma:%p", dbFName, tbName, int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) {
meta->tableType, meta->suid, tbMeta); int32_t code = 0;
STableMeta *tblMeta = NULL;
SCtgTbMetaCtx tbCtx = {0};
tbCtx.flag = CTG_FLAG_UNKNOWN_STB;
tbCtx.pName = pTableName;
SSTableMetaVersion metaRent = { CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &tbCtx, &tblMeta));
.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion};
strcpy(metaRent.dbFName, dbFName);
strcpy(metaRent.stbName, tbName);
CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion)));
if (NULL == tblMeta) {
ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
}
int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
*dst = taosMemoryMalloc(sizeof(SDBVgInfo));
if (NULL == *dst) {
qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
} }
memcpy(*dst, src, sizeof(SDBVgInfo)); char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
size_t hashSize = taosHashGetSize(src->vgHash); if (TSDB_SUPER_TABLE == tblMeta->tableType) {
(*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); CTG_ERR_JRET(ctgPutRmStbToQueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq));
if (NULL == (*dst)->vgHash) { } else {
qError("taosHashInit %d failed", (int32_t)hashSize); CTG_ERR_JRET(ctgPutRmTbToQueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq));
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
} }
int32_t *vgId = NULL; _return:
void *pIter = taosHashIterate(src->vgHash, NULL);
while (pIter) {
vgId = taosHashGetKey(pIter, NULL);
if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) {
qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize);
taosHashCancelIterate(src->vgHash, pIter);
taosHashCleanup((*dst)->vgHash);
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
pIter = taosHashIterate(src->vgHash, pIter); taosMemoryFreeClear(tblMeta);
}
return TSDB_CODE_SUCCESS; CTG_RET(code);
} }
int32_t ctgGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SCtgDBCache **dbCache, int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) {
SDBVgInfo **pInfo) {
bool inCache = false;
int32_t code = 0; int32_t code = 0;
CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache));
if (inCache) { if (*dbCache) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -1789,19 +68,11 @@ int32_t ctgGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const ...@@ -1789,19 +68,11 @@ int32_t ctgGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const
tstrncpy(input.db, dbFName, tListLen(input.db)); tstrncpy(input.db, dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION; input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, &DbOut, NULL));
if (code) {
if (CTG_DB_NOT_EXIST(code) && input.vgVersion > CTG_DEFAULT_INVALID_VERSION) {
ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId);
ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId);
}
CTG_ERR_RET(code);
}
CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo));
CTG_ERR_RET(ctgPushUpdateVgMsgInQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, false)); CTG_ERR_RET(ctgPutUpdateVgToQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, false));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -1813,18 +84,17 @@ _return: ...@@ -1813,18 +84,17 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgRefreshDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName) { int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) {
bool inCache = false;
int32_t code = 0; int32_t code = 0;
SCtgDBCache *dbCache = NULL; SCtgDBCache* dbCache = NULL;
CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache));
SUseDbOutput DbOut = {0}; SUseDbOutput DbOut = {0};
SBuildUseDBInput input = {0}; SBuildUseDBInput input = {0};
tstrncpy(input.db, dbFName, tListLen(input.db)); tstrncpy(input.db, dbFName, tListLen(input.db));
if (inCache) { if (NULL != dbCache) {
input.dbId = dbCache->dbId; input.dbId = dbCache->dbId;
ctgReleaseVgInfo(dbCache); ctgReleaseVgInfo(dbCache);
...@@ -1834,56 +104,29 @@ int32_t ctgRefreshDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, c ...@@ -1834,56 +104,29 @@ int32_t ctgRefreshDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, c
input.vgVersion = CTG_DEFAULT_INVALID_VERSION; input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
input.numOfTable = 0; input.numOfTable = 0;
code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); code = ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, &DbOut, NULL);
if (code) { if (code) {
if (CTG_DB_NOT_EXIST(code) && inCache) { if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) {
ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId);
ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId); ctgPutRmDBToQueue(pCtg, input.db, input.dbId);
} }
CTG_ERR_RET(code); CTG_ERR_RET(code);
} }
CTG_ERR_RET(ctgPushUpdateVgMsgInQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, true)); CTG_ERR_RET(ctgPutUpdateVgToQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, true));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) {
*pOutput = taosMemoryMalloc(sizeof(STableMetaOutput));
if (NULL == *pOutput) {
qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(*pOutput, output, sizeof(STableMetaOutput));
if (output->tbMeta) {
int32_t metaSize = CTG_META_SIZE(output->tbMeta);
(*pOutput)->tbMeta = taosMemoryMalloc(metaSize);
if (NULL == (*pOutput)->tbMeta) {
qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
taosMemoryFreeClear(*pOutput);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize);
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgRefreshTblMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, int32_t flag,
STableMetaOutput **pOutput, bool syncReq) {
if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOutput, bool syncReq) {
SVgroupInfo vgroupInfo = {0}; SVgroupInfo vgroupInfo = {0};
int32_t code = 0; int32_t code = 0;
if (!CTG_FLAG_IS_SYS_DB(flag)) { if (!CTG_FLAG_IS_SYS_DB(ctx->flag)) {
CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); CTG_ERR_RET(catalogGetTableHashVgroup(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo));
} }
STableMetaOutput moutput = {0}; STableMetaOutput moutput = {0};
...@@ -1893,539 +136,243 @@ int32_t ctgRefreshTblMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, ...@@ -1893,539 +136,243 @@ int32_t ctgRefreshTblMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps,
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
} }
if (CTG_FLAG_IS_SYS_DB(flag)) { if (CTG_FLAG_IS_SYS_DB(ctx->flag)) {
ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName));
CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), (char *)ctx->pName->dbname, (char *)ctx->pName->tname, output, NULL));
(char *)pTableName->tname, output)); } else if (CTG_FLAG_IS_STB(ctx->flag)) {
} else if (CTG_FLAG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName));
ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName));
// if get from mnode failed, will not try vnode // if get from mnode failed, will not try vnode
CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTrans, pMgmtEps, pTableName, output)); CTG_ERR_JRET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, output, NULL));
if (CTG_IS_META_NULL(output->metaType)) { if (CTG_IS_META_NULL(output->metaType)) {
CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo, output, NULL));
} }
} else { } else {
ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pTableName), flag); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag);
// if get from vnode failed or no table meta, will not try mnode // if get from vnode failed or no table meta, will not try mnode
CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo, output, NULL));
if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) {
ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName));
taosMemoryFreeClear(output->tbMeta);
CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output));
} else if (CTG_IS_META_BOTH(output->metaType)) {
int32_t exist = 0;
if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) {
CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist));
}
if (0 == exist) {
CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput));
if (CTG_IS_META_NULL(moutput.metaType)) {
SET_META_TYPE_NULL(output->metaType);
}
taosMemoryFreeClear(output->tbMeta);
output->tbMeta = moutput.tbMeta;
moutput.tbMeta = NULL;
} else {
taosMemoryFreeClear(output->tbMeta);
SET_META_TYPE_CTABLE(output->metaType);
}
}
}
if (CTG_IS_META_NULL(output->metaType)) {
ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName));
catalogRemoveTableMeta(pCtg, pTableName);
CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
}
if (CTG_IS_META_TABLE(output->metaType)) {
ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName,
output->tbMeta->tableType);
} else {
ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName,
output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType));
}
if (pOutput) {
CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput));
}
CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, syncReq));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(output->tbMeta);
taosMemoryFreeClear(output);
CTG_RET(code);
}
int32_t ctgGetTableMeta(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName,
STableMeta **pTableMeta, int32_t flag) {
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
bool inCache = false;
int32_t code = 0;
uint64_t dbId = 0;
uint64_t suid = 0;
STableMetaOutput *output = NULL;
if (CTG_IS_SYS_DBNAME(pTableName->dbname)) {
CTG_FLAG_SET_SYS_DB(flag);
}
CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId));
int32_t tbType = 0;
if (inCache) {
if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) &&
((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_SYS_DB(flag)))) {
goto _return;
}
tbType = (*pTableMeta)->tableType;
suid = (*pTableMeta)->suid;
taosMemoryFreeClear(*pTableMeta);
}
if (CTG_FLAG_IS_UNKNOWN_STB(flag)) {
CTG_FLAG_SET_STB(flag, tbType);
}
while (true) {
CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output, false));
if (CTG_IS_META_TABLE(output->metaType)) {
*pTableMeta = output->tbMeta;
goto _return;
}
if (CTG_IS_META_BOTH(output->metaType)) {
memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta));
*pTableMeta = output->tbMeta;
goto _return;
}
if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) {
ctgError("invalid metaType:%d", output->metaType);
taosMemoryFreeClear(output->tbMeta);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
// HANDLE ONLY CHILD TABLE META
SName stbName = *pTableName;
strcpy(stbName.tname, output->tbName);
taosMemoryFreeClear(output->tbMeta);
CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL));
if (!inCache) {
ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname);
continue;
}
memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta));
break;
}
_return:
if (CTG_TABLE_NOT_EXIST(code) && inCache) {
char dbFName[TSDB_DB_FNAME_LEN] = {0};
if (CTG_FLAG_IS_SYS_DB(flag)) {
strcpy(dbFName, pTableName->dbname);
} else {
tNameGetFullDbName(pTableName, dbFName);
}
if (TSDB_SUPER_TABLE == tbType) {
ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid, false);
} else {
ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, false);
}
}
taosMemoryFreeClear(output);
if (*pTableMeta) {
ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType);
ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta);
}
CTG_RET(code);
}
int32_t ctgChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, const char *dbFName,
AUTH_TYPE type, bool *pass) {
bool inCache = false;
int32_t code = 0;
*pass = false;
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass));
if (inCache) {
return TSDB_CODE_SUCCESS;
}
SGetUserAuthRsp authRsp = {0};
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp));
if (authRsp.superAuth) {
*pass = true;
goto _return;
}
if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) {
*pass = true;
goto _return;
}
if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) {
*pass = true;
}
if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) {
*pass = true;
}
_return:
ctgPushUpdateUserMsgInQueue(pCtg, &authRsp, false);
return TSDB_CODE_SUCCESS;
}
int32_t ctgActUpdateVg(SCtgMetaAction *action) {
int32_t code = 0;
SCtgUpdateVgMsg *msg = action->data;
CTG_ERR_JRET(ctgUpdateDBVgInfo(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo));
_return:
ctgFreeVgInfo(msg->dbInfo);
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgActRemoveDB(SCtgMetaAction *action) {
int32_t code = 0;
SCtgRemoveDBMsg *msg = action->data;
SCatalog *pCtg = msg->pCtg;
SCtgDBCache *dbCache = NULL;
ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache);
if (NULL == dbCache) {
goto _return;
}
if (dbCache->dbId != msg->dbId) {
ctgInfo("dbId already updated, dbFName:%s, dbId:%" PRIx64 ", targetId:%" PRIx64, msg->dbFName, dbCache->dbId,
msg->dbId);
goto _return;
}
CTG_ERR_JRET(ctgRemoveDB(pCtg, dbCache, msg->dbFName));
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgActUpdateTbl(SCtgMetaAction *action) {
int32_t code = 0;
SCtgUpdateTblMsg *msg = action->data;
SCatalog *pCtg = msg->pCtg;
STableMetaOutput *output = msg->output;
SCtgDBCache *dbCache = NULL;
if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) {
ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) {
ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache));
if (NULL == dbCache) {
ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, output->dbFName, output->dbId);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) {
int32_t metaSize = CTG_META_SIZE(output->tbMeta);
CTG_ERR_JRET(
ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize));
}
if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) {
CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName,
(STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta)));
}
_return:
if (output) {
taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output->tbMeta);
taosMemoryFreeClear(output);
CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), output->dbFName, output->tbName, output, NULL));
} else if (CTG_IS_META_BOTH(output->metaType)) {
int32_t exist = 0;
if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) {
CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist));
} }
taosMemoryFreeClear(msg); if (0 == exist) {
CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), output->dbFName, output->tbName, &moutput, NULL));
CTG_RET(code); if (CTG_IS_META_NULL(moutput.metaType)) {
} SET_META_TYPE_NULL(output->metaType);
}
int32_t ctgActRemoveStb(SCtgMetaAction *action) { taosMemoryFreeClear(output->tbMeta);
int32_t code = 0; output->tbMeta = moutput.tbMeta;
SCtgRemoveStbMsg *msg = action->data; moutput.tbMeta = NULL;
SCatalog *pCtg = msg->pCtg; } else {
taosMemoryFreeClear(output->tbMeta);
SCtgDBCache *dbCache = NULL; SET_META_TYPE_CTABLE(output->metaType);
ctgGetDBCache(pCtg, msg->dbFName, &dbCache); }
if (NULL == dbCache) {
return TSDB_CODE_SUCCESS;
} }
if (msg->dbId && (dbCache->dbId != msg->dbId)) {
ctgDebug("dbId already modified, dbFName:%s, current:%" PRIx64 ", dbId:%" PRIx64 ", stb:%s, suid:%" PRIx64,
msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid);
return TSDB_CODE_SUCCESS;
} }
CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); if (CTG_IS_META_NULL(output->metaType)) {
if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName));
ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false);
msg->suid); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
} else {
CTG_CACHE_STAT_SUB(stblNum, 1);
} }
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); if (CTG_IS_META_TABLE(output->metaType)) {
if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, output->tbMeta->tableType);
ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid);
} else { } else {
CTG_CACHE_STAT_SUB(tblNum, 1); ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType));
} }
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock);
ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); if (pOutput) {
CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput));
}
CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, output, syncReq));
ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); return TSDB_CODE_SUCCESS;
_return: _return:
taosMemoryFreeClear(msg); taosMemoryFreeClear(output->tbMeta);
taosMemoryFreeClear(output);
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgActRemoveTbl(SCtgMetaAction *action) { int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) {
int32_t code = 0; if (CTG_IS_SYS_DBNAME(ctx->pName->dbname)) {
SCtgRemoveTblMsg *msg = action->data; CTG_FLAG_SET_SYS_DB(ctx->flag);
SCatalog *pCtg = msg->pCtg;
SCtgDBCache *dbCache = NULL;
ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
if (NULL == dbCache) {
return TSDB_CODE_SUCCESS;
} }
if (dbCache->dbId != msg->dbId) { CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, ctx, pTableMeta));
ctgDebug("dbId already modified, dbFName:%s, current:%" PRIx64 ", dbId:%" PRIx64 ", tbName:%s", msg->dbFName,
dbCache->dbId, msg->dbId, msg->tbName); if (*pTableMeta) {
if (CTG_FLAG_MATCH_STB(ctx->flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) || (CTG_FLAG_IS_SYS_DB(ctx->flag)))) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); taosMemoryFreeClear(*pTableMeta);
if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
} else {
CTG_CACHE_STAT_SUB(tblNum, 1);
} }
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); if (CTG_FLAG_IS_UNKNOWN_STB(ctx->flag)) {
CTG_FLAG_SET_STB(ctx->flag, ctx->tbInfo.tbType);
_return: }
taosMemoryFreeClear(msg);
CTG_RET(code); return TSDB_CODE_SUCCESS;
} }
int32_t ctgActUpdateUser(SCtgMetaAction *action) {
int32_t ctgGetTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) {
int32_t code = 0; int32_t code = 0;
SCtgUpdateUserMsg *msg = action->data; STableMetaOutput *output = NULL;
SCatalog *pCtg = msg->pCtg;
if (NULL == pCtg->userCache) { CTG_ERR_RET(ctgGetTbMetaFromCache(CTG_PARAMS_LIST(), ctx, pTableMeta));
pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), if (*pTableMeta) {
false, HASH_ENTRY_LOCK); goto _return;
if (NULL == pCtg->userCache) {
ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); while (true) {
if (NULL == pUser) { CTG_ERR_JRET(ctgRefreshTbMeta(CTG_PARAMS_LIST(), ctx, &output, false));
SCtgUserAuth userAuth = {0};
userAuth.version = msg->userAuth.version;
userAuth.superUser = msg->userAuth.superAuth;
userAuth.createdDbs = msg->userAuth.createdDbs;
userAuth.readDbs = msg->userAuth.readDbs;
userAuth.writeDbs = msg->userAuth.writeDbs;
if (taosHashPut(pCtg->userCache, msg->userAuth.user, sizeof(msg->userAuth.user), &userAuth, sizeof(userAuth))) { if (CTG_IS_META_TABLE(output->metaType)) {
ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); *pTableMeta = output->tbMeta;
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); goto _return;
} }
taosMemoryFreeClear(msg); if (CTG_IS_META_BOTH(output->metaType)) {
memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta));
return TSDB_CODE_SUCCESS; *pTableMeta = output->tbMeta;
goto _return;
} }
pUser->version = msg->userAuth.version; if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) {
ctgError("invalid metaType:%d", output->metaType);
CTG_LOCK(CTG_WRITE, &pUser->lock); taosMemoryFreeClear(output->tbMeta);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
taosHashCleanup(pUser->createdDbs); }
pUser->createdDbs = msg->userAuth.createdDbs;
msg->userAuth.createdDbs = NULL;
taosHashCleanup(pUser->readDbs); // HANDLE ONLY CHILD TABLE META
pUser->readDbs = msg->userAuth.readDbs;
msg->userAuth.readDbs = NULL;
taosHashCleanup(pUser->writeDbs); taosMemoryFreeClear(output->tbMeta);
pUser->writeDbs = msg->userAuth.writeDbs;
msg->userAuth.writeDbs = NULL;
CTG_UNLOCK(CTG_WRITE, &pUser->lock); SName stbName = *ctx->pName;
strcpy(stbName.tname, output->tbName);
SCtgTbMetaCtx stbCtx = {0};
stbCtx.flag = ctx->flag;
stbCtx.pName = &stbName;
_return: CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, pTableMeta));
if (NULL == *pTableMeta) {
ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, ctx->pName->tname);
continue;
}
taosHashCleanup(msg->userAuth.createdDbs); memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta));
taosHashCleanup(msg->userAuth.readDbs);
taosHashCleanup(msg->userAuth.writeDbs);
taosMemoryFreeClear(msg); break;
}
CTG_RET(code); _return:
}
void *ctgUpdateThreadFunc(void *param) { if (CTG_TABLE_NOT_EXIST(code) && ctx->tbInfo.inCache) {
setThreadName("catalog"); char dbFName[TSDB_DB_FNAME_LEN] = {0};
if (CTG_FLAG_IS_SYS_DB(ctx->flag)) {
strcpy(dbFName, ctx->pName->dbname);
} else {
tNameGetFullDbName(ctx->pName, dbFName);
}
qInfo("catalog update thread started"); if (TSDB_SUPER_TABLE == ctx->tbInfo.tbType) {
ctgPutRmStbToQueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false);
} else {
ctgPutRmTbToQueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false);
}
}
CTG_LOCK(CTG_READ, &gCtgMgmt.lock); taosMemoryFreeClear(output);
while (true) { if (*pTableMeta) {
if (tsem_wait(&gCtgMgmt.queue.reqSem)) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", ctx->pName->tname, (*pTableMeta)->tableType);
qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); ctgdShowTableMeta(pCtg, ctx->pName->tname, *pTableMeta);
} }
if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) { CTG_RET(code);
tsem_post(&gCtgMgmt.queue.rspSem); }
break;
}
SCtgMetaAction *action = NULL;
ctgPopAction(&action);
SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg;
ctgDebug("process [%s] action", gCtgAction[action->act].name); int32_t ctgChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) {
bool inCache = false;
int32_t code = 0;
(*gCtgAction[action->act].func)(action); *pass = false;
gCtgMgmt.queue.seqDone = action->seqId; CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass));
if (action->syncReq) { if (inCache) {
tsem_post(&gCtgMgmt.queue.rspSem); return TSDB_CODE_SUCCESS;
} }
CTG_RUNTIME_STAT_ADD(qDoneNum, 1); SGetUserAuthRsp authRsp = {0};
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(CTG_PARAMS_LIST(), user, &authRsp, NULL));
ctgdShowClusterCache(pCtg); if (authRsp.superAuth) {
*pass = true;
goto _return;
} }
CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) {
*pass = true;
qInfo("catalog update thread stopped"); goto _return;
}
return NULL; if (type == AUTH_TYPE_READ && authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName))) {
} *pass = true;
} else if (type == AUTH_TYPE_WRITE && authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName))) {
*pass = true;
}
int32_t ctgStartUpdateThread() { _return:
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { ctgPutUpdateUserToQueue(pCtg, &authRsp, false);
terrno = TAOS_SYSTEM_ERROR(errno);
CTG_ERR_RET(terrno);
}
taosThreadAttrDestroy(&thAttr);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SName* pTableName, SArray** pVgList) {
SArray **pVgList) {
STableMeta *tbMeta = NULL; STableMeta *tbMeta = NULL;
int32_t code = 0; int32_t code = 0;
SVgroupInfo vgroupInfo = {0}; SVgroupInfo vgroupInfo = {0};
SCtgDBCache *dbCache = NULL; SCtgDBCache* dbCache = NULL;
SArray *vgList = NULL; SArray *vgList = NULL;
SDBVgInfo *vgInfo = NULL; SDBVgInfo *vgInfo = NULL;
SCtgTbMetaCtx ctx = {0};
ctx.pName = pTableName;
ctx.flag = CTG_FLAG_UNKNOWN_STB;
*pVgList = NULL; *pVgList = NULL;
CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); CTG_ERR_JRET(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, &tbMeta));
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db); tNameGetFullDbName(pTableName, db);
SHashObj *vgHash = NULL; SHashObj *vgHash = NULL;
CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, &dbCache, &vgInfo)); CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, &dbCache, &vgInfo));
if (dbCache) { if (dbCache) {
vgHash = dbCache->vgInfo->vgHash; vgHash = dbCache->vgInfo->vgHash;
...@@ -2491,7 +438,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { ...@@ -2491,7 +438,7 @@ int32_t catalogInit(SCatalogCfg *cfg) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
atomic_store_8((int8_t *)&gCtgMgmt.exit, false); atomic_store_8((int8_t*)&gCtgMgmt.exit, false);
if (cfg) { if (cfg) {
memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg));
...@@ -2518,8 +465,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { ...@@ -2518,8 +465,7 @@ int32_t catalogInit(SCatalogCfg *cfg) {
gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND;
} }
gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
false, HASH_ENTRY_LOCK);
if (NULL == gCtgMgmt.pCluster) { if (NULL == gCtgMgmt.pCluster) {
qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
...@@ -2542,21 +488,26 @@ int32_t catalogInit(SCatalogCfg *cfg) { ...@@ -2542,21 +488,26 @@ int32_t catalogInit(SCatalogCfg *cfg) {
} }
gCtgMgmt.queue.tail = gCtgMgmt.queue.head; gCtgMgmt.queue.tail = gCtgMgmt.queue.head;
gCtgMgmt.jobPool = taosOpenRef(200, ctgFreeJob);
if (gCtgMgmt.jobPool < 0) {
qError("taosOpenRef failed, error:%s", tstrerror(terrno));
CTG_ERR_RET(terrno);
}
CTG_ERR_RET(ctgStartUpdateThread()); CTG_ERR_RET(ctgStartUpdateThread());
qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec);
gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
if (NULL == catalogHandle) { if (NULL == catalogHandle) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
if (NULL == gCtgMgmt.pCluster) { if (NULL == gCtgMgmt.pCluster) {
qError("catalog cluster cache are not ready, clusterId:%" PRIx64, clusterId); qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId);
CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY);
} }
...@@ -2564,11 +515,11 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { ...@@ -2564,11 +515,11 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) {
SCatalog *clusterCtg = NULL; SCatalog *clusterCtg = NULL;
while (true) { while (true) {
SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char *)&clusterId, sizeof(clusterId)); SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId));
if (ctg && (*ctg)) { if (ctg && (*ctg)) {
*catalogHandle = *ctg; *catalogHandle = *ctg;
qDebug("got catalog handle from cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, *ctg); qDebug("got catalog handle from cache, clusterId:%"PRIx64", CTG:%p", clusterId, *ctg);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -2583,13 +534,18 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { ...@@ -2583,13 +534,18 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) {
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE));
clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
false, HASH_ENTRY_LOCK);
if (NULL == clusterCtg->dbCache) { if (NULL == clusterCtg->dbCache) {
qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
} }
SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == metaCache) {
qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES);
if (code) { if (code) {
if (HASH_NODE_EXIST(code)) { if (HASH_NODE_EXIST(code)) {
...@@ -2597,11 +553,11 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { ...@@ -2597,11 +553,11 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) {
continue; continue;
} }
qError("taosHashPut CTG to cache failed, clusterId:%" PRIx64, clusterId); qError("taosHashPut CTG to cache failed, clusterId:%"PRIx64, clusterId);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
qDebug("add CTG to cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, clusterCtg); qDebug("add CTG to cache, clusterId:%"PRIx64", CTG:%p", clusterId, clusterCtg);
break; break;
} }
...@@ -2619,13 +575,13 @@ _return: ...@@ -2619,13 +575,13 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
void catalogFreeHandle(SCatalog *pCtg) { void catalogFreeHandle(SCatalog* pCtg) {
if (NULL == pCtg) { if (NULL == pCtg) {
return; return;
} }
if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) {
ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%" PRIx64, pCtg->clusterId); ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId);
return; return;
} }
...@@ -2635,10 +591,10 @@ void catalogFreeHandle(SCatalog *pCtg) { ...@@ -2635,10 +591,10 @@ void catalogFreeHandle(SCatalog *pCtg) {
ctgFreeHandle(pCtg); ctgFreeHandle(pCtg);
ctgInfo("handle freed, culsterId:%" PRIx64, clusterId); ctgInfo("handle freed, culsterId:%"PRIx64, clusterId);
} }
int32_t catalogGetDBVgVersion(SCatalog *pCtg, const char *dbFName, int32_t *version, int64_t *dbId, int32_t *tableNum) { int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) {
...@@ -2646,11 +602,10 @@ int32_t catalogGetDBVgVersion(SCatalog *pCtg, const char *dbFName, int32_t *vers ...@@ -2646,11 +602,10 @@ int32_t catalogGetDBVgVersion(SCatalog *pCtg, const char *dbFName, int32_t *vers
} }
SCtgDBCache *dbCache = NULL; SCtgDBCache *dbCache = NULL;
bool inCache = false;
int32_t code = 0; int32_t code = 0;
CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache));
if (!inCache) { if (NULL == dbCache) {
*version = CTG_DEFAULT_INVALID_VERSION; *version = CTG_DEFAULT_INVALID_VERSION;
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
} }
...@@ -2671,20 +626,19 @@ _return: ...@@ -2671,20 +626,19 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) {
SArray **vgroupList) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { if (NULL == pCtg || NULL == dbFName || NULL == pTrans || NULL == pMgmtEps || NULL == vgroupList) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
SCtgDBCache *dbCache = NULL; SCtgDBCache* dbCache = NULL;
int32_t code = 0; int32_t code = 0;
SArray *vgList = NULL; SArray *vgList = NULL;
SHashObj *vgHash = NULL; SHashObj *vgHash = NULL;
SDBVgInfo *vgInfo = NULL; SDBVgInfo *vgInfo = NULL;
CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, &dbCache, &vgInfo)); CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName, &dbCache, &vgInfo));
if (dbCache) { if (dbCache) {
vgHash = dbCache->vgInfo->vgHash; vgHash = dbCache->vgInfo->vgHash;
} else { } else {
...@@ -2711,7 +665,8 @@ _return: ...@@ -2711,7 +665,8 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SDBVgInfo *dbInfo) {
int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) {
CTG_API_ENTER(); CTG_API_ENTER();
int32_t code = 0; int32_t code = 0;
...@@ -2721,14 +676,15 @@ int32_t catalogUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId ...@@ -2721,14 +676,15 @@ int32_t catalogUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId
CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT);
} }
code = ctgPushUpdateVgMsgInQueue(pCtg, dbFName, dbId, dbInfo, false); code = ctgPutUpdateVgToQueue(pCtg, dbFName, dbId, dbInfo, false);
_return: _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogRemoveDB(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) {
CTG_API_ENTER(); CTG_API_ENTER();
int32_t code = 0; int32_t code = 0;
...@@ -2741,7 +697,7 @@ int32_t catalogRemoveDB(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ...@@ -2741,7 +697,7 @@ int32_t catalogRemoveDB(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
} }
CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); CTG_ERR_JRET(ctgPutRmDBToQueue(pCtg, dbFName, dbId));
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
...@@ -2750,9 +706,11 @@ _return: ...@@ -2750,9 +706,11 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogUpdateVgEpSet(SCatalog *pCtg, const char *dbFName, int32_t vgId, SEpSet *epSet) { return 0; } int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet) {
return 0;
}
int32_t catalogRemoveTableMeta(SCatalog *pCtg, const SName *pTableName) { int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) {
CTG_API_ENTER(); CTG_API_ENTER();
int32_t code = 0; int32_t code = 0;
...@@ -2765,35 +723,15 @@ int32_t catalogRemoveTableMeta(SCatalog *pCtg, const SName *pTableName) { ...@@ -2765,35 +723,15 @@ int32_t catalogRemoveTableMeta(SCatalog *pCtg, const SName *pTableName) {
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
} }
STableMeta *tblMeta = NULL; CTG_ERR_JRET(ctgRemoveTbMetaFromCache(pCtg, pTableName, true));
bool inCache = false;
uint64_t dbId = 0;
CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId));
if (!inCache) {
ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
goto _return;
}
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
if (TSDB_SUPER_TABLE == tblMeta->tableType) {
CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid, true));
} else {
CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true));
}
ctgDebug("table meta %s.%s removed", dbFName, pTableName->tname);
_return: _return:
taosMemoryFreeClear(tblMeta);
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogRemoveStbMeta(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *stbName, uint64_t suid) {
int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) {
CTG_API_ENTER(); CTG_API_ENTER();
int32_t code = 0; int32_t code = 0;
...@@ -2806,7 +744,7 @@ int32_t catalogRemoveStbMeta(SCatalog *pCtg, const char *dbFName, uint64_t dbId, ...@@ -2806,7 +744,7 @@ int32_t catalogRemoveStbMeta(SCatalog *pCtg, const char *dbFName, uint64_t dbId,
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
} }
CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid, true)); CTG_ERR_JRET(ctgPutRmStbToQueue(pCtg, dbFName, dbId, stbName, suid, true));
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
...@@ -2815,26 +753,27 @@ _return: ...@@ -2815,26 +753,27 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogGetIndexMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
const char *pIndexName, SIndexMeta **pIndexMeta) {
return 0;
}
int32_t catalogGetTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName,
STableMeta **pTableMeta) {
CTG_API_ENTER(); CTG_API_ENTER();
CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_UNKNOWN_STB)); SCtgTbMetaCtx ctx = {0};
ctx.pName = (SName*)pTableName;
ctx.flag = CTG_FLAG_UNKNOWN_STB;
CTG_API_LEAVE(ctgGetTbMeta(pCtg, pTrans, pMgmtEps, &ctx, pTableMeta));
} }
int32_t catalogGetSTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
STableMeta **pTableMeta) {
CTG_API_ENTER(); CTG_API_ENTER();
CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_STB)); SCtgTbMetaCtx ctx = {0};
ctx.pName = (SName*)pTableName;
ctx.flag = CTG_FLAG_STB;
CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta));
} }
int32_t catalogUpdateSTableMeta(SCatalog *pCtg, STableMetaRsp *rspMsg) { int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == rspMsg) { if (NULL == pCtg || NULL == rspMsg) {
...@@ -2858,7 +797,7 @@ int32_t catalogUpdateSTableMeta(SCatalog *pCtg, STableMetaRsp *rspMsg) { ...@@ -2858,7 +797,7 @@ int32_t catalogUpdateSTableMeta(SCatalog *pCtg, STableMetaRsp *rspMsg) {
CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta));
CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, false)); CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, output, false));
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
...@@ -2870,86 +809,7 @@ _return: ...@@ -2870,86 +809,7 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t ctgGetTbSverFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tbType, uint64_t *suid, int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) {
char *stbName) {
*sver = -1;
if (NULL == pCtg->dbCache) {
ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname);
return TSDB_CODE_SUCCESS;
}
SCtgDBCache *dbCache = NULL;
char dbFName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, dbFName);
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
ctgDebug("db %s not in cache", pTableName->tname);
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname));
if (tbMeta) {
*tbType = tbMeta->tableType;
*suid = tbMeta->suid;
if (*tbType != TSDB_CHILD_TABLE) {
*sver = tbMeta->sversion;
}
}
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == tbMeta) {
ctgReleaseDBCache(pCtg, dbCache);
return TSDB_CODE_SUCCESS;
}
if (*tbType != TSDB_CHILD_TABLE) {
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname);
return TSDB_CODE_SUCCESS;
}
ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid);
CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock);
STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid));
if (NULL == stbMeta || NULL == *stbMeta) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid);
return TSDB_CODE_SUCCESS;
}
if ((*stbMeta)->suid != *suid) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid,
(*stbMeta)->suid);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
size_t nameLen = 0;
char *name = taosHashGetKey(*stbMeta, &nameLen);
strncpy(stbName, name, nameLen);
stbName[nameLen] = 0;
*sver = (*stbMeta)->sversion;
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname);
return TSDB_CODE_SUCCESS;
}
int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, SArray *pTables) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) {
...@@ -2974,7 +834,7 @@ int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgm ...@@ -2974,7 +834,7 @@ int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgm
int32_t tbType = 0; int32_t tbType = 0;
uint64_t suid = 0; uint64_t suid = 0;
char stbName[TSDB_TABLE_FNAME_LEN]; char stbName[TSDB_TABLE_FNAME_LEN];
ctgGetTbSverFromCache(pCtg, &name, &sver, &tbType, &suid, stbName); ctgReadTbSverFromCache(pCtg, &name, &sver, &tbType, &suid, stbName);
if (sver >= 0 && sver < pTb->sver) { if (sver >= 0 && sver < pTb->sver) {
switch (tbType) { switch (tbType) {
case TSDB_CHILD_TABLE: { case TSDB_CHILD_TABLE: {
...@@ -2997,7 +857,8 @@ int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgm ...@@ -2997,7 +857,8 @@ int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgm
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
} }
int32_t catalogRefreshDBVgInfo(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const char *dbFName) {
int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName) {
...@@ -3007,31 +868,34 @@ int32_t catalogRefreshDBVgInfo(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmt ...@@ -3007,31 +868,34 @@ int32_t catalogRefreshDBVgInfo(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmt
CTG_API_LEAVE(ctgRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName)); CTG_API_LEAVE(ctgRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName));
} }
int32_t catalogRefreshTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) {
int32_t isSTable) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, SCtgTbMetaCtx ctx = {0};
CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL, true)); ctx.pName = (SName*)pTableName;
ctx.flag = CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable);
CTG_API_LEAVE(ctgRefreshTbMeta(CTG_PARAMS_LIST(), &ctx, NULL, true));
} }
int32_t catalogRefreshGetTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) {
STableMeta **pTableMeta, int32_t isSTable) {
CTG_API_ENTER(); CTG_API_ENTER();
CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, SCtgTbMetaCtx ctx = {0};
CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); ctx.pName = (SName*)pTableName;
ctx.flag = CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable);
CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta));
} }
int32_t catalogGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) {
SArray **pVgList) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
...@@ -3040,33 +904,11 @@ int32_t catalogGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgm ...@@ -3040,33 +904,11 @@ int32_t catalogGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgm
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
int32_t code = 0; CTG_API_LEAVE(ctgGetTbDistVgInfo(pCtg, pTrans, pMgmtEps, (SName*)pTableName, pVgList));
while (true) {
code = ctgGetTableDistVgInfo(pCtg, pRpc, pMgmtEps, pTableName, pVgList);
if (code) {
if (TSDB_CODE_CTG_VG_META_MISMATCH == code) {
CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName,
CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(CTG_FLAG_UNKNOWN_STB), NULL, true));
char dbFName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, dbFName);
CTG_ERR_JRET(ctgRefreshDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName));
continue;
}
}
break;
}
_return:
CTG_API_LEAVE(code);
} }
int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName,
SVgroupInfo *pVgroup) { int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) {
CTG_API_ENTER(); CTG_API_ENTER();
if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { if (CTG_IS_SYS_DBNAME(pTableName->dbname)) {
...@@ -3074,7 +916,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pM ...@@ -3074,7 +916,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pM
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
SCtgDBCache *dbCache = NULL; SCtgDBCache* dbCache = NULL;
int32_t code = 0; int32_t code = 0;
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db); tNameGetFullDbName(pTableName, db);
...@@ -3099,8 +941,8 @@ _return: ...@@ -3099,8 +941,8 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SCatalogReq *pReq,
SMetaData *pRsp) { int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) {
...@@ -3110,8 +952,8 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, ...@@ -3110,8 +952,8 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps,
int32_t code = 0; int32_t code = 0;
pRsp->pTableMeta = NULL; pRsp->pTableMeta = NULL;
if (pReq->pTableName) { if (pReq->pTableMeta) {
int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName); int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableMeta);
if (tbNum <= 0) { if (tbNum <= 0) {
ctgError("empty table name list, tbNum:%d", tbNum); ctgError("empty table name list, tbNum:%d", tbNum);
CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT);
...@@ -3124,10 +966,13 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, ...@@ -3124,10 +966,13 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps,
} }
for (int32_t i = 0; i < tbNum; ++i) { for (int32_t i = 0; i < tbNum; ++i) {
SName *name = taosArrayGet(pReq->pTableName, i); SName *name = taosArrayGet(pReq->pTableMeta, i);
STableMeta *pTableMeta = NULL; STableMeta *pTableMeta = NULL;
SCtgTbMetaCtx ctx = {0};
ctx.pName = name;
ctx.flag = CTG_FLAG_UNKNOWN_STB;
CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); CTG_ERR_JRET(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, &pTableMeta));
if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) {
ctgError("taosArrayPush failed, idx:%d", i); ctgError("taosArrayPush failed, idx:%d", i);
...@@ -3139,7 +984,7 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, ...@@ -3139,7 +984,7 @@ int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps,
if (pReq->qNodeRequired) { if (pReq->qNodeRequired) {
pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr)); pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr));
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, pRsp->pQnodeList)); CTG_ERR_JRET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), pRsp->pQnodeList, NULL));
} }
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
...@@ -3160,22 +1005,50 @@ _return: ...@@ -3160,22 +1005,50 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogGetQnodeList(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SArray *pQnodeList) { int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == fp || NULL == param) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
SCtgJob *pJob = NULL;
CTG_ERR_JRET(ctgInitJob(CTG_PARAMS_LIST(), &pJob, reqId, pReq, fp, param));
CTG_ERR_JRET(ctgLaunchJob(pJob));
*jobId = pJob->refId;
_return:
if (pJob) {
taosReleaseRef(gCtgMgmt.jobPool, pJob->refId);
if (code) {
taosRemoveRef(gCtgMgmt.jobPool, pJob->refId);
}
}
CTG_API_LEAVE(code);
}
int32_t catalogGetQnodeList(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pQnodeList) {
CTG_API_ENTER(); CTG_API_ENTER();
int32_t code = 0; int32_t code = 0;
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pQnodeList) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, pQnodeList)); CTG_ERR_JRET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), pQnodeList, NULL));
_return: _return:
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
} }
int32_t catalogGetExpiredSTables(SCatalog *pCtg, SSTableMetaVersion **stables, uint32_t *num) { int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == stables || NULL == num) { if (NULL == pCtg || NULL == stables || NULL == num) {
...@@ -3185,7 +1058,7 @@ int32_t catalogGetExpiredSTables(SCatalog *pCtg, SSTableMetaVersion **stables, u ...@@ -3185,7 +1058,7 @@ int32_t catalogGetExpiredSTables(SCatalog *pCtg, SSTableMetaVersion **stables, u
CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion)));
} }
int32_t catalogGetExpiredDBs(SCatalog *pCtg, SDbVgVersion **dbs, uint32_t *num) { int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == dbs || NULL == num) { if (NULL == pCtg || NULL == dbs || NULL == num) {
...@@ -3195,7 +1068,7 @@ int32_t catalogGetExpiredDBs(SCatalog *pCtg, SDbVgVersion **dbs, uint32_t *num) ...@@ -3195,7 +1068,7 @@ int32_t catalogGetExpiredDBs(SCatalog *pCtg, SDbVgVersion **dbs, uint32_t *num)
CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion)));
} }
int32_t catalogGetExpiredUsers(SCatalog *pCtg, SUserAuthVersion **users, uint32_t *num) { int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == users || NULL == num) { if (NULL == pCtg || NULL == users || NULL == num) {
...@@ -3214,94 +1087,88 @@ int32_t catalogGetExpiredUsers(SCatalog *pCtg, SUserAuthVersion **users, uint32_ ...@@ -3214,94 +1087,88 @@ int32_t catalogGetExpiredUsers(SCatalog *pCtg, SUserAuthVersion **users, uint32_
uint32_t i = 0; uint32_t i = 0;
SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL);
while (pAuth != NULL) { while (pAuth != NULL) {
void *key = taosHashGetKey(pAuth, NULL); size_t len = 0;
strncpy((*users)[i].user, key, sizeof((*users)[i].user)); void *key = taosHashGetKey(pAuth, &len);
strncpy((*users)[i].user, key, len);
(*users)[i].user[len] = 0;
(*users)[i].version = pAuth->version; (*users)[i].version = pAuth->version;
++i;
pAuth = taosHashIterate(pCtg->userCache, pAuth); pAuth = taosHashIterate(pCtg->userCache, pAuth);
} }
CTG_API_LEAVE(TSDB_CODE_SUCCESS); CTG_API_LEAVE(TSDB_CODE_SUCCESS);
} }
int32_t catalogGetDBCfg(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SDbCfgInfo *pDbCfg) {
int32_t catalogGetDBCfg(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pRpc, pMgmtEps, dbFName, pDbCfg)); CTG_API_LEAVE(ctgGetDBCfgFromMnode(CTG_PARAMS_LIST(), dbFName, pDbCfg, NULL));
} }
int32_t catalogGetIndexInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *indexName, int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo) {
SIndexInfo *pInfo) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo)); CTG_API_LEAVE(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), indexName, pInfo, NULL));
} }
int32_t catalogGetUdfInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *funcName, SFuncInfo **pInfo) { int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
int32_t code = 0; int32_t code = 0;
*pInfo = taosMemoryMalloc(sizeof(SFuncInfo)); CTG_ERR_JRET(ctgGetUdfInfoFromMnode(CTG_PARAMS_LIST(), funcName, pInfo, NULL));
if (NULL == *pInfo) {
CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo));
_return: _return:
if (code) {
taosMemoryFreeClear(*pInfo);
}
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, const char *dbFName, int32_t catalogChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) {
AUTH_TYPE type, bool *pass) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
int32_t code = 0; int32_t code = 0;
CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); CTG_ERR_JRET(ctgChkAuth(CTG_PARAMS_LIST(), user, dbFName, type, pass));
_return: _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t catalogUpdateUserAuthInfo(SCatalog *pCtg, SGetUserAuthRsp *pAuth) { int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) {
CTG_API_ENTER(); CTG_API_ENTER();
if (NULL == pCtg || NULL == pAuth) { if (NULL == pCtg || NULL == pAuth) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); CTG_API_LEAVE(ctgPutUpdateUserToQueue(pCtg, pAuth, false));
} }
void catalogDestroy(void) { void catalogDestroy(void) {
qInfo("start to destroy catalog"); qInfo("start to destroy catalog");
if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t *)&gCtgMgmt.exit)) { if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) {
return; return;
} }
atomic_store_8((int8_t *)&gCtgMgmt.exit, true); atomic_store_8((int8_t*)&gCtgMgmt.exit, true);
if (tsem_post(&gCtgMgmt.queue.reqSem)) { if (tsem_post(&gCtgMgmt.queue.reqSem)) {
qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
...@@ -3337,3 +1204,5 @@ void catalogDestroy(void) { ...@@ -3337,3 +1204,5 @@ void catalogDestroy(void) {
qInfo("catalog destroyed"); qInfo("catalog destroyed");
} }
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "trpc.h"
#include "query.h"
#include "tname.h"
#include "catalogInt.h"
#include "systable.h"
#include "tref.h"
int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx);
pTask->type = CTG_TASK_GET_TB_META;
pTask->taskId = taskIdx;
pTask->pJob = pJob;
pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetaCtx));
if (NULL == pTask->taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgTbMetaCtx* ctx = pTask->taskCtx;
ctx->pName = taosMemoryMalloc(sizeof(*name));
if (NULL == ctx->pName) {
taosMemoryFree(pTask->taskCtx);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(ctx->pName, name, sizeof(*name));
ctx->flag = CTG_FLAG_UNKNOWN_STB;
qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, pTask->type, name->tname);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx);
pTask->type = CTG_TASK_GET_DB_VGROUP;
pTask->taskId = taskIdx;
pTask->pJob = pJob;
pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgDbVgCtx));
if (NULL == pTask->taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgDbVgCtx* ctx = pTask->taskCtx;
memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName));
qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, pTask->type, dbFName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx);
pTask->type = CTG_TASK_GET_DB_CFG;
pTask->taskId = taskIdx;
pTask->pJob = pJob;
pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgDbCfgCtx));
if (NULL == pTask->taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgDbCfgCtx* ctx = pTask->taskCtx;
memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName));
qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, pTask->type, dbFName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx);
pTask->type = CTG_TASK_GET_TB_HASH;
pTask->taskId = taskIdx;
pTask->pJob = pJob;
pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashCtx));
if (NULL == pTask->taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgTbHashCtx* ctx = pTask->taskCtx;
ctx->pName = taosMemoryMalloc(sizeof(*name));
if (NULL == ctx->pName) {
taosMemoryFree(pTask->taskCtx);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(ctx->pName, name, sizeof(*name));
tNameGetFullDbName(ctx->pName, ctx->dbFName);
qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, pTask->type, name->tname);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx);
pTask->type = CTG_TASK_GET_QNODE;
pTask->taskId = taskIdx;
pTask->pJob = pJob;
pTask->taskCtx = NULL;
qDebug("QID:%" PRIx64 " task %d type %d initialized", pJob->queryId, taskIdx, pTask->type);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, char *name) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx);
pTask->type = CTG_TASK_GET_INDEX;
pTask->taskId = taskIdx;
pTask->pJob = pJob;
pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgIndexCtx));
if (NULL == pTask->taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgIndexCtx* ctx = pTask->taskCtx;
strcpy(ctx->indexFName, name);
qDebug("QID:%" PRIx64 " task %d type %d initialized, indexFName:%s", pJob->queryId, taskIdx, pTask->type, name);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, char *name) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx);
pTask->type = CTG_TASK_GET_UDF;
pTask->taskId = taskIdx;
pTask->pJob = pJob;
pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgUdfCtx));
if (NULL == pTask->taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgUdfCtx* ctx = pTask->taskCtx;
strcpy(ctx->udfName, name);
qDebug("QID:%" PRIx64 " task %d type %d initialized, udfName:%s", pJob->queryId, taskIdx, pTask->type, name);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx);
pTask->type = CTG_TASK_GET_USER;
pTask->taskId = taskIdx;
pTask->pJob = pJob;
pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgUserCtx));
if (NULL == pTask->taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgUserCtx* ctx = pTask->taskCtx;
memcpy(&ctx->user, user, sizeof(*user));
qDebug("QID:%" PRIx64 " task %d type %d initialized, user:%s", pJob->queryId, taskIdx, pTask->type, user->user);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param) {
int32_t code = 0;
int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta);
int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup);
int32_t tbHashNum = (int32_t)taosArrayGetSize(pReq->pTableHash);
int32_t udfNum = (int32_t)taosArrayGetSize(pReq->pUdf);
int32_t qnodeNum = pReq->qNodeRequired ? 1 : 0;
int32_t dbCfgNum = (int32_t)taosArrayGetSize(pReq->pDbCfg);
int32_t indexNum = (int32_t)taosArrayGetSize(pReq->pIndex);
int32_t userNum = (int32_t)taosArrayGetSize(pReq->pUser);
int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum;
if (taskNum <= 0) {
ctgError("empty input for job, taskNum:%d", taskNum);
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
*job = taosMemoryCalloc(1, sizeof(SCtgJob));
if (NULL == *job) {
ctgError("calloc %d failed", (int32_t)sizeof(SCtgJob));
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgJob *pJob = *job;
pJob->queryId = reqId;
pJob->userFp = fp;
pJob->pCtg = pCtg;
pJob->pTrans = pTrans;
pJob->pMgmtEps = pMgmtEps;
pJob->userParam = param;
pJob->tbMetaNum = tbMetaNum;
pJob->tbHashNum = tbHashNum;
pJob->qnodeNum = qnodeNum;
pJob->dbVgNum = dbVgNum;
pJob->udfNum = udfNum;
pJob->dbCfgNum = dbCfgNum;
pJob->indexNum = indexNum;
pJob->userNum = userNum;
pJob->pTasks = taosArrayInit(taskNum, sizeof(SCtgTask));
if (NULL == pJob->pTasks) {
ctgError("taosArrayInit %d tasks failed", taskNum);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t taskIdx = 0;
for (int32_t i = 0; i < dbVgNum; ++i) {
char *dbFName = taosArrayGet(pReq->pDbVgroup, i);
CTG_ERR_JRET(ctgInitGetDbVgTask(pJob, taskIdx++, dbFName));
}
for (int32_t i = 0; i < dbCfgNum; ++i) {
char *dbFName = taosArrayGet(pReq->pDbCfg, i);
CTG_ERR_JRET(ctgInitGetDbCfgTask(pJob, taskIdx++, dbFName));
}
for (int32_t i = 0; i < tbMetaNum; ++i) {
SName *name = taosArrayGet(pReq->pTableMeta, i);
CTG_ERR_JRET(ctgInitGetTbMetaTask(pJob, taskIdx++, name));
}
for (int32_t i = 0; i < tbHashNum; ++i) {
SName *name = taosArrayGet(pReq->pTableHash, i);
CTG_ERR_JRET(ctgInitGetTbHashTask(pJob, taskIdx++, name));
}
for (int32_t i = 0; i < indexNum; ++i) {
char *indexName = taosArrayGet(pReq->pIndex, i);
CTG_ERR_JRET(ctgInitGetIndexTask(pJob, taskIdx++, indexName));
}
for (int32_t i = 0; i < udfNum; ++i) {
char *udfName = taosArrayGet(pReq->pUdf, i);
CTG_ERR_JRET(ctgInitGetUdfTask(pJob, taskIdx++, udfName));
}
for (int32_t i = 0; i < userNum; ++i) {
SUserAuthInfo *user = taosArrayGet(pReq->pUser, i);
CTG_ERR_JRET(ctgInitGetUserTask(pJob, taskIdx++, user));
}
if (qnodeNum) {
CTG_ERR_JRET(ctgInitGetQnodeTask(pJob, taskIdx++));
}
pJob->refId = taosAddRef(gCtgMgmt.jobPool, pJob);
if (pJob->refId < 0) {
ctgError("add job to ref failed, error: %s", tstrerror(terrno));
CTG_ERR_JRET(terrno);
}
taosAcquireRef(gCtgMgmt.jobPool, pJob->refId);
qDebug("QID:%" PRIx64 ", job %" PRIx64 " initialized, task num %d", pJob->queryId, pJob->refId, taskNum);
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(*job);
CTG_RET(code);
}
int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableMeta) {
pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(STableMeta));
if (NULL == pJob->jobRes.pTableMeta) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pJob->jobRes.pTableMeta, pTask->res);
taosMemoryFreeClear(pTask->res);
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDbVgroup) {
pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, POINTER_BYTES);
if (NULL == pJob->jobRes.pDbVgroup) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pJob->jobRes.pDbVgroup, &pTask->res);
pTask->res = NULL;
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableHash) {
pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SVgroupInfo));
if (NULL == pJob->jobRes.pTableHash) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pJob->jobRes.pTableHash, &pTask->res);
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpIndexRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pIndex) {
pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SIndexInfo));
if (NULL == pJob->jobRes.pIndex) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pJob->jobRes.pIndex, pTask->res);
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
TSWAP(pJob->jobRes.pQnodeList, pTask->res);
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDbCfg) {
pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SDbCfgInfo));
if (NULL == pJob->jobRes.pDbCfg) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pJob->jobRes.pDbCfg, &pTask->res);
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpUdfRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pUdfList) {
pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SFuncInfo));
if (NULL == pJob->jobRes.pUdfList) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pJob->jobRes.pUdfList, pTask->res);
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpUserRes(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pUser) {
pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(bool));
if (NULL == pJob->jobRes.pUser) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pJob->jobRes.pUser, pTask->res);
return TSDB_CODE_SUCCESS;
}
int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
SCtgJob* pJob = pTask->pJob;
int32_t code = 0;
qDebug("QID:%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
if (rspCode) {
int32_t lastCode = atomic_val_compare_exchange_32(&pJob->rspCode, 0, rspCode);
if (0 == lastCode) {
CTG_ERR_JRET(rspCode);
}
return TSDB_CODE_SUCCESS;
}
int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1);
if (taskDone < taosArrayGetSize(pJob->pTasks)) {
qDebug("task done: %d, total: %d", taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
return TSDB_CODE_SUCCESS;
}
CTG_ERR_JRET(ctgMakeAsyncRes(pJob));
_return:
qDebug("QID:%" PRIx64 " user callback with rsp %s", pJob->queryId, tstrerror(code));
(*pJob->userFp)(&pJob->jobRes, pJob->userParam, code);
taosRemoveRef(gCtgMgmt.jobPool, pJob->refId);
CTG_RET(code);
}
int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
SCtgDBCache *dbCache = NULL;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
switch (reqType) {
case TDMT_MND_USE_DB: {
SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out;
SVgroupInfo vgInfo = {0};
CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, &vgInfo));
ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag);
CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask));
return TSDB_CODE_SUCCESS;
}
case TDMT_MND_TABLE_META: {
STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out;
if (CTG_IS_META_NULL(pOut->metaType)) {
if (CTG_FLAG_IS_STB(ctx->flag)) {
char dbFName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(ctx->pName, dbFName);
CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache));
if (NULL != dbCache) {
SVgroupInfo vgInfo = {0};
CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo));
ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag);
CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask));
ctgReleaseVgInfo(dbCache);
ctgReleaseDBCache(pCtg, dbCache);
} else {
SBuildUseDBInput input = {0};
tstrncpy(input.db, dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask));
}
return TSDB_CODE_SUCCESS;
}
ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName));
ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false);
CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
}
if (pTask->msgCtx.lastOut) {
TSWAP(pTask->msgCtx.out, pTask->msgCtx.lastOut);
STableMetaOutput* pLastOut = (STableMetaOutput*)pTask->msgCtx.out;
TSWAP(pLastOut->tbMeta, pOut->tbMeta);
}
break;
}
case TDMT_VND_TABLE_META: {
STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out;
if (CTG_IS_META_NULL(pOut->metaType)) {
ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName));
ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false);
CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
}
if (CTG_FLAG_IS_STB(ctx->flag)) {
break;
}
if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) {
ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName));
taosMemoryFreeClear(pOut->tbMeta);
CTG_ERR_JRET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, NULL, pTask));
} else if (CTG_IS_META_BOTH(pOut->metaType)) {
int32_t exist = 0;
if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) {
CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pOut->dbFName, pOut->tbName, &exist));
}
if (0 == exist) {
TSWAP(pTask->msgCtx.lastOut, pTask->msgCtx.out);
CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), pOut->dbFName, pOut->tbName, NULL, pTask));
} else {
taosMemoryFreeClear(pOut->tbMeta);
SET_META_TYPE_CTABLE(pOut->metaType);
}
}
break;
}
default:
ctgError("invalid reqType %d", reqType);
CTG_ERR_JRET(TSDB_CODE_INVALID_MSG);
break;
}
STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out;
ctgUpdateTbMetaToCache(pCtg, pOut, false);
if (CTG_IS_META_BOTH(pOut->metaType)) {
memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta));
} else if (CTG_IS_META_CTABLE(pOut->metaType)) {
SName stbName = *ctx->pName;
strcpy(stbName.tname, pOut->tbName);
SCtgTbMetaCtx stbCtx = {0};
stbCtx.flag = ctx->flag;
stbCtx.pName = &stbName;
CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta));
if (NULL == pOut->tbMeta) {
ctgDebug("stb no longer exist, stbName:%s", stbName.tname);
CTG_ERR_JRET(ctgRelaunchGetTbMetaTask(pTask));
return TSDB_CODE_SUCCESS;
}
memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta));
}
TSWAP(pTask->res, pOut->tbMeta);
_return:
if (dbCache) {
ctgReleaseVgInfo(dbCache);
ctgReleaseDBCache(pCtg, dbCache);
}
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx;
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
switch (reqType) {
case TDMT_MND_USE_DB: {
SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out;
CTG_ERR_JRET(ctgGenerateVgList(pCtg, pOut->dbVgroup->vgHash, (SArray**)&pTask->res));
CTG_ERR_JRET(ctgPutUpdateVgToQueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false));
pOut->dbVgroup = NULL;
break;
}
default:
ctgError("invalid reqType %d", reqType);
CTG_ERR_JRET(TSDB_CODE_INVALID_MSG);
break;
}
_return:
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgHandleGetTbHashRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx;
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
switch (reqType) {
case TDMT_MND_USE_DB: {
SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out;
pTask->res = taosMemoryMalloc(sizeof(SVgroupInfo));
if (NULL == pTask->res) {
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, (SVgroupInfo*)pTask->res));
CTG_ERR_JRET(ctgPutUpdateVgToQueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false));
pOut->dbVgroup = NULL;
break;
}
default:
ctgError("invalid reqType %d", reqType);
CTG_ERR_JRET(TSDB_CODE_INVALID_MSG);
break;
}
_return:
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
TSWAP(pTask->res, pTask->msgCtx.out);
_return:
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgHandleGetQnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
TSWAP(pTask->res, pTask->msgCtx.out);
_return:
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgHandleGetIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
TSWAP(pTask->res, pTask->msgCtx.out);
_return:
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgHandleGetUdfRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
TSWAP(pTask->res, pTask->msgCtx.out);
_return:
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgHandleGetUserRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
SCtgDBCache *dbCache = NULL;
CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx;
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
bool pass = false;
SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out;
if (pOut->superAuth) {
pass = true;
goto _return;
}
if (pOut->createdDbs && taosHashGet(pOut->createdDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
pass = true;
goto _return;
}
if (ctx->user.type == AUTH_TYPE_READ && pOut->readDbs && taosHashGet(pOut->readDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
pass = true;
} else if (ctx->user.type == AUTH_TYPE_WRITE && pOut->writeDbs && taosHashGet(pOut->writeDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
pass = true;
}
_return:
if (TSDB_CODE_SUCCESS == code) {
pTask->res = taosMemoryCalloc(1, sizeof(bool));
if (NULL == pTask->res) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
*(bool*)pTask->res = pass;
}
}
ctgPutUpdateUserToQueue(pCtg, pOut, false);
pTask->msgCtx.out = NULL;
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
int32_t code = 0;
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
if (CTG_FLAG_IS_SYS_DB(ctx->flag)) {
ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(ctx->pName));
CTG_RET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), (char *)ctx->pName->dbname, (char *)ctx->pName->tname, NULL, pTask));
}
if (CTG_FLAG_IS_STB(ctx->flag)) {
ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName));
// if get from mnode failed, will not try vnode
CTG_RET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, NULL, pTask));
}
SCtgDBCache *dbCache = NULL;
char dbFName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(ctx->pName, dbFName);
CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache));
if (NULL == dbCache) {
SVgroupInfo vgInfo = {0};
CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo));
ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag);
CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask));
} else {
SBuildUseDBInput input = {0};
tstrncpy(input.db, dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask));
}
_return:
if (dbCache) {
ctgReleaseVgInfo(dbCache);
ctgReleaseDBCache(pCtg, dbCache);
}
CTG_RET(code);
}
int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
CTG_ERR_RET(ctgGetTbMetaFromCache(CTG_PARAMS_LIST(), (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res));
if (pTask->res) {
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask));
return TSDB_CODE_SUCCESS;
}
int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) {
int32_t code = 0;
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
SCtgDBCache *dbCache = NULL;
SCtgDbVgCtx* pCtx = (SCtgDbVgCtx*)pTask->taskCtx;
CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache));
if (NULL != dbCache) {
CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgInfo->vgHash, (SArray**)&pTask->res));
CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0));
} else {
SBuildUseDBInput input = {0};
tstrncpy(input.db, pCtx->dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask));
}
_return:
if (dbCache) {
ctgReleaseVgInfo(dbCache);
ctgReleaseDBCache(pCtg, dbCache);
}
CTG_RET(code);
}
int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) {
int32_t code = 0;
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
SCtgDBCache *dbCache = NULL;
SCtgTbHashCtx* pCtx = (SCtgTbHashCtx*)pTask->taskCtx;
CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache));
if (NULL != dbCache) {
pTask->res = taosMemoryMalloc(sizeof(SVgroupInfo));
if (NULL == pTask->res) {
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res));
CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0));
} else {
SBuildUseDBInput input = {0};
tstrncpy(input.db, pCtx->dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask));
}
_return:
if (dbCache) {
ctgReleaseVgInfo(dbCache);
ctgReleaseDBCache(pCtg, dbCache);
}
CTG_RET(code);
}
int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
CTG_ERR_RET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), NULL, pTask));
return TSDB_CODE_SUCCESS;
}
int32_t ctgLaunchGetDbCfgTask(SCtgTask *pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx;
CTG_ERR_RET(ctgGetDBCfgFromMnode(CTG_PARAMS_LIST(), pCtx->dbFName, NULL, pTask));
return TSDB_CODE_SUCCESS;
}
int32_t ctgLaunchGetIndexTask(SCtgTask *pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
SCtgIndexCtx* pCtx = (SCtgIndexCtx*)pTask->taskCtx;
CTG_ERR_RET(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), pCtx->indexFName, NULL, pTask));
return TSDB_CODE_SUCCESS;
}
int32_t ctgLaunchGetUdfTask(SCtgTask *pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
SCtgUdfCtx* pCtx = (SCtgUdfCtx*)pTask->taskCtx;
CTG_ERR_RET(ctgGetUdfInfoFromMnode(CTG_PARAMS_LIST(), pCtx->udfName, NULL, pTask));
return TSDB_CODE_SUCCESS;
}
int32_t ctgLaunchGetUserTask(SCtgTask *pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
void *pTrans = pTask->pJob->pTrans;
const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps;
SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx;
bool inCache = false;
bool pass = false;
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pCtx->user.user, pCtx->user.dbFName, pCtx->user.type, &inCache, &pass));
if (inCache) {
pTask->res = taosMemoryCalloc(1, sizeof(bool));
if (NULL == pTask->res) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
*(bool*)pTask->res = pass;
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(CTG_PARAMS_LIST(), pCtx->user.user, NULL, pTask));
return TSDB_CODE_SUCCESS;
}
int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask) {
ctgResetTbMetaTask(pTask);
CTG_ERR_RET(ctgLaunchGetTbMetaTask(pTask));
return TSDB_CODE_SUCCESS;
}
SCtgAsyncFps gCtgAsyncFps[] = {
{ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes},
{ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes},
{ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes},
{ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes},
{ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes},
{ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes},
{ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes},
{ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes},
};
int32_t ctgMakeAsyncRes(SCtgJob *pJob) {
int32_t code = 0;
int32_t taskNum = taosArrayGetSize(pJob->pTasks);
for (int32_t i = 0; i < taskNum; ++i) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, i);
CTG_ERR_RET((*gCtgAsyncFps[pTask->type].dumpResFp)(pTask));
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgLaunchJob(SCtgJob *pJob) {
int32_t taskNum = taosArrayGetSize(pJob->pTasks);
for (int32_t i = 0; i < taskNum; ++i) {
SCtgTask *pTask = taosArrayGet(pJob->pTasks, i);
qDebug("QID:%" PRIx64 " start to launch task %d", pJob->queryId, pTask->taskId);
CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask));
}
return TSDB_CODE_SUCCESS;
}
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "trpc.h"
#include "query.h"
#include "tname.h"
#include "catalogInt.h"
#include "systable.h"
SCtgAction gCtgAction[CTG_ACT_MAX] = {
{
CTG_ACT_UPDATE_VG,
"update vgInfo",
ctgActUpdateVg
},
{
CTG_ACT_UPDATE_TBL,
"update tbMeta",
ctgActUpdateTb
},
{
CTG_ACT_REMOVE_DB,
"remove DB",
ctgActRemoveDB
},
{
CTG_ACT_REMOVE_STB,
"remove stbMeta",
ctgActRemoveStb
},
{
CTG_ACT_REMOVE_TBL,
"remove tbMeta",
ctgActRemoveTb
},
{
CTG_ACT_UPDATE_USER,
"update user",
ctgActUpdateUser
}
};
int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) {
CTG_LOCK(CTG_READ, &dbCache->vgLock);
if (dbCache->deleted) {
CTG_UNLOCK(CTG_READ, &dbCache->vgLock);
ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId);
*inCache = false;
return TSDB_CODE_SUCCESS;
}
if (NULL == dbCache->vgInfo) {
CTG_UNLOCK(CTG_READ, &dbCache->vgLock);
*inCache = false;
ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId);
return TSDB_CODE_SUCCESS;
}
*inCache = true;
return TSDB_CODE_SUCCESS;
}
int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) {
CTG_LOCK(CTG_WRITE, &dbCache->vgLock);
if (dbCache->deleted) {
ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId);
CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock);
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
return TSDB_CODE_SUCCESS;
}
void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) {
taosHashRelease(pCtg->dbCache, dbCache);
}
void ctgReleaseVgInfo(SCtgDBCache *dbCache) {
CTG_UNLOCK(CTG_READ, &dbCache->vgLock);
}
void ctgWReleaseVgInfo(SCtgDBCache *dbCache) {
CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock);
}
int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) {
char *p = strchr(dbFName, '.');
if (p && CTG_IS_SYS_DBNAME(p + 1)) {
dbFName = p + 1;
}
SCtgDBCache *dbCache = NULL;
if (acquire) {
dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName));
} else {
dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName));
}
if (NULL == dbCache) {
*pCache = NULL;
ctgDebug("db not in cache, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
}
if (dbCache->deleted) {
if (acquire) {
ctgReleaseDBCache(pCtg, dbCache);
}
*pCache = NULL;
ctgDebug("db is removing from cache, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
}
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) {
CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true));
}
int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) {
CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false));
}
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) {
SCtgDBCache *dbCache = NULL;
if (NULL == pCtg->dbCache) {
ctgDebug("empty db cache, dbFName:%s", dbFName);
goto _return;
}
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
ctgDebug("db %s not in cache", dbFName);
goto _return;
}
bool inCache = false;
ctgAcquireVgInfo(pCtg, dbCache, &inCache);
if (!inCache) {
ctgDebug("vgInfo of db %s not in cache", dbFName);
goto _return;
}
*pCache = dbCache;
CTG_CACHE_STAT_ADD(vgHitNum, 1);
ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName);
return TSDB_CODE_SUCCESS;
_return:
if (dbCache) {
ctgReleaseDBCache(pCtg, dbCache);
}
*pCache = NULL;
CTG_CACHE_STAT_ADD(vgMissNum, 1);
return TSDB_CODE_SUCCESS;
}
int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) {
if (NULL == pCtg->dbCache) {
*exist = 0;
ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName);
return TSDB_CODE_SUCCESS;
}
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
size_t sz = 0;
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName));
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == tbMeta) {
ctgReleaseDBCache(pCtg, dbCache);
*exist = 0;
ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName);
return TSDB_CODE_SUCCESS;
}
*exist = 1;
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName);
return TSDB_CODE_SUCCESS;
}
int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) {
int32_t code = 0;
SCtgDBCache *dbCache = NULL;
*pTableMeta = NULL;
if (NULL == pCtg->dbCache) {
ctgDebug("empty tbmeta cache, tbName:%s", ctx->pName->tname);
return TSDB_CODE_SUCCESS;
}
char dbFName[TSDB_DB_FNAME_LEN] = {0};
if (CTG_FLAG_IS_SYS_DB(ctx->flag)) {
strcpy(dbFName, ctx->pName->dbname);
} else {
tNameGetFullDbName(ctx->pName, dbFName);
}
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
ctgDebug("db %s not in cache", ctx->pName->tname);
return TSDB_CODE_SUCCESS;
}
int32_t sz = 0;
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
taosHashGetDup_m(dbCache->tbCache.metaCache, ctx->pName->tname, strlen(ctx->pName->tname), (void **)pTableMeta, &sz);
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == *pTableMeta) {
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname);
return TSDB_CODE_SUCCESS;
}
STableMeta* tbMeta = *pTableMeta;
ctx->tbInfo.inCache = true;
ctx->tbInfo.dbId = dbCache->dbId;
ctx->tbInfo.suid = tbMeta->suid;
ctx->tbInfo.tbType = tbMeta->tableType;
if (tbMeta->tableType != TSDB_CHILD_TABLE) {
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, ctx->pName->tname);
CTG_CACHE_STAT_ADD(tblHitNum, 1);
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock);
STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid));
if (NULL == stbMeta || NULL == *stbMeta) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid);
goto _return;
}
if ((*stbMeta)->suid != tbMeta->suid) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t metaSize = CTG_META_SIZE(*stbMeta);
*pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize);
if (NULL == *pTableMeta) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgError("realloc size[%d] failed", metaSize);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta));
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
CTG_CACHE_STAT_ADD(tblHitNum, 1);
ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname);
return TSDB_CODE_SUCCESS;
_return:
ctgReleaseDBCache(pCtg, dbCache);
taosMemoryFreeClear(*pTableMeta);
CTG_CACHE_STAT_ADD(tblMissNum, 1);
CTG_RET(code);
}
int32_t ctgReadTbSverFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tbType, uint64_t *suid,
char *stbName) {
*sver = -1;
if (NULL == pCtg->dbCache) {
ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname);
return TSDB_CODE_SUCCESS;
}
SCtgDBCache *dbCache = NULL;
char dbFName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, dbFName);
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
ctgDebug("db %s not in cache", pTableName->tname);
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname));
if (tbMeta) {
*tbType = tbMeta->tableType;
*suid = tbMeta->suid;
if (*tbType != TSDB_CHILD_TABLE) {
*sver = tbMeta->sversion;
}
}
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == tbMeta) {
ctgReleaseDBCache(pCtg, dbCache);
return TSDB_CODE_SUCCESS;
}
if (*tbType != TSDB_CHILD_TABLE) {
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname);
return TSDB_CODE_SUCCESS;
}
ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid);
CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock);
STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid));
if (NULL == stbMeta || NULL == *stbMeta) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid);
return TSDB_CODE_SUCCESS;
}
if ((*stbMeta)->suid != *suid) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid,
(*stbMeta)->suid);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
size_t nameLen = 0;
char *name = taosHashGetKey(*stbMeta, &nameLen);
strncpy(stbName, name, nameLen);
stbName[nameLen] = 0;
*sver = (*stbMeta)->sversion;
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) {
if (NULL == pCtg->dbCache) {
ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName);
return TSDB_CODE_SUCCESS;
}
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName));
if (NULL == pTableMeta) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName);
ctgReleaseDBCache(pCtg, dbCache);
return TSDB_CODE_SUCCESS;
}
*tbType = atomic_load_8(&pTableMeta->tableType);
taosHashRelease(dbCache->tbCache.metaCache, pTableMeta);
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgReleaseDBCache(pCtg, dbCache);
ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType);
return TSDB_CODE_SUCCESS;
}
int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) {
if (NULL == pCtg->userCache) {
ctgDebug("empty user auth cache, user:%s", user);
goto _return;
}
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user));
if (NULL == pUser) {
ctgDebug("user not in cache, user:%s", user);
goto _return;
}
*inCache = true;
ctgDebug("Got user from cache, user:%s", user);
CTG_CACHE_STAT_ADD(userHitNum, 1);
if (pUser->superUser) {
*pass = true;
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &pUser->lock);
if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) {
*pass = true;
CTG_UNLOCK(CTG_READ, &pUser->lock);
return TSDB_CODE_SUCCESS;
}
if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) {
*pass = true;
}
if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) {
*pass = true;
}
CTG_UNLOCK(CTG_READ, &pUser->lock);
return TSDB_CODE_SUCCESS;
_return:
*inCache = false;
CTG_CACHE_STAT_ADD(userMissNum, 1);
return TSDB_CODE_SUCCESS;
}
void ctgWaitAction(SCtgMetaAction *action) {
while (true) {
tsem_wait(&gCtgMgmt.queue.rspSem);
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) {
tsem_post(&gCtgMgmt.queue.rspSem);
break;
}
if (gCtgMgmt.queue.seqDone >= action->seqId) {
break;
}
tsem_post(&gCtgMgmt.queue.rspSem);
sched_yield();
}
}
void ctgPopAction(SCtgMetaAction **action) {
SCtgQNode *orig = gCtgMgmt.queue.head;
SCtgQNode *node = gCtgMgmt.queue.head->next;
gCtgMgmt.queue.head = gCtgMgmt.queue.head->next;
CTG_QUEUE_SUB();
taosMemoryFreeClear(orig);
*action = &node->action;
}
int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) {
SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode));
if (NULL == node) {
qError("calloc %d failed", (int32_t)sizeof(SCtgQNode));
CTG_RET(TSDB_CODE_CTG_MEM_ERROR);
}
action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1);
node->action = *action;
CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
gCtgMgmt.queue.tail->next = node;
gCtgMgmt.queue.tail = node;
CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
CTG_QUEUE_ADD();
CTG_RUNTIME_STAT_ADD(qNum, 1);
tsem_post(&gCtgMgmt.queue.reqSem);
ctgDebug("action [%s] added into queue", gCtgAction[action->act].name);
if (action->syncReq) {
ctgWaitAction(action);
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgPutRmDBToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) {
int32_t code = 0;
SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB};
SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
char *p = strchr(dbFName, '.');
if (p && CTG_IS_SYS_DBNAME(p + 1)) {
dbFName = p + 1;
}
msg->pCtg = pCtg;
strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
msg->dbId = dbId;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(action.data);
CTG_RET(code);
}
int32_t ctgPutRmStbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq};
SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
msg->pCtg = pCtg;
strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
strncpy(msg->stbName, stbName, sizeof(msg->stbName));
msg->dbId = dbId;
msg->suid = suid;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(action.data);
CTG_RET(code);
}
int32_t ctgPutRmTbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq};
SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
msg->pCtg = pCtg;
strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
strncpy(msg->tbName, tbName, sizeof(msg->tbName));
msg->dbId = dbId;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(action.data);
CTG_RET(code);
}
int32_t ctgPutUpdateVgToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq};
SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg));
ctgFreeVgInfo(dbInfo);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
char *p = strchr(dbFName, '.');
if (p && CTG_IS_SYS_DBNAME(p + 1)) {
dbFName = p + 1;
}
strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
msg->pCtg = pCtg;
msg->dbId = dbId;
msg->dbInfo = dbInfo;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
ctgFreeVgInfo(dbInfo);
taosMemoryFreeClear(action.data);
CTG_RET(code);
}
int32_t ctgPutUpdateTbToQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq};
SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
char *p = strchr(output->dbFName, '.');
if (p && CTG_IS_SYS_DBNAME(p + 1)) {
memmove(output->dbFName, p + 1, strlen(p + 1));
}
msg->pCtg = pCtg;
msg->output = output;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgPutUpdateUserToQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) {
int32_t code = 0;
SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq};
SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
msg->pCtg = pCtg;
msg->userAuth = *pAuth;
action.data = msg;
CTG_ERR_JRET(ctgPushAction(pCtg, &action));
return TSDB_CODE_SUCCESS;
_return:
tFreeSGetUserAuthRsp(pAuth);
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) {
mgmt->slotRIdx = 0;
mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND;
mgmt->type = type;
size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum;
mgmt->slots = taosMemoryCalloc(1, msgSize);
if (NULL == mgmt->slots) {
qError("calloc %d failed", (int32_t)msgSize);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum);
return TSDB_CODE_SUCCESS;
}
int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size);
if (NULL == slot->meta) {
qError("taosArrayInit %d failed, id:%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
}
if (NULL == taosArrayPush(slot->meta, meta)) {
qError("taosArrayPush meta to rent failed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
slot->needSort = true;
qDebug("add meta to rent, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (slot->needSort) {
qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta));
taosArraySort(slot->meta, sortCompare);
slot->needSort = false;
qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta));
}
void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ);
if (NULL == orig) {
qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
memcpy(orig, meta, size);
qDebug("meta in rent updated, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
if (code) {
qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type);
CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size));
}
CTG_RET(code);
}
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (slot->needSort) {
taosArraySort(slot->meta, sortCompare);
slot->needSort = false;
qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type);
}
int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ);
if (idx < 0) {
qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
taosArrayRemove(slot->meta, idx);
qDebug("meta in rent removed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) {
int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1);
if (ridx >= mgmt->slotNum) {
ridx %= mgmt->slotNum;
atomic_store_16(&mgmt->slotRIdx, ridx);
}
SCtgRentSlot *slot = &mgmt->slots[ridx];
int32_t code = 0;
CTG_LOCK(CTG_READ, &slot->lock);
if (NULL == slot->meta) {
qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type);
*num = 0;
goto _return;
}
size_t metaNum = taosArrayGetSize(slot->meta);
if (metaNum <= 0) {
qDebug("no meta in slot:%d, type:%d", ridx, mgmt->type);
*num = 0;
goto _return;
}
size_t msize = metaNum * size;
*res = taosMemoryMalloc(msize);
if (NULL == *res) {
qError("malloc %d failed", (int32_t)msize);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
void *meta = taosArrayGet(slot->meta, 0);
memcpy(*res, meta, msize);
*num = (uint32_t)metaNum;
qDebug("Got %d meta from rent, type:%d", (int32_t)metaNum, mgmt->type);
_return:
CTG_UNLOCK(CTG_READ, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) {
while (true) {
int64_t msec = taosGetTimestampMs();
int64_t lsec = atomic_load_64(&mgmt->lastReadMsec);
if ((msec - lsec) < CTG_RENT_SLOT_SECOND * 1000) {
*res = NULL;
*num = 0;
qDebug("too short time period to get expired meta, type:%d", mgmt->type);
return TSDB_CODE_SUCCESS;
}
if (lsec != atomic_val_compare_exchange_64(&mgmt->lastReadMsec, lsec, msec)) {
continue;
}
break;
}
CTG_ERR_RET(ctgMetaRentGetImpl(mgmt, res, num, size));
return TSDB_CODE_SUCCESS;
}
int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
int32_t code = 0;
SCtgDBCache newDBCache = {0};
newDBCache.dbId = dbId;
newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == newDBCache.tbCache.metaCache) {
ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
if (NULL == newDBCache.tbCache.stbCache) {
ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache));
if (code) {
if (HASH_NODE_EXIST(code)) {
ctgDebug("db already in cache, dbFName:%s", dbFName);
goto _return;
}
ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
CTG_CACHE_STAT_ADD(dbNum, 1);
SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1};
strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId);
CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion)));
ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId);
return TSDB_CODE_SUCCESS;
_return:
ctgFreeDbCache(&newDBCache);
CTG_RET(code);
}
void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) {
CTG_LOCK(CTG_WRITE, &cache->stbLock);
if (cache->stbCache) {
void *pIter = taosHashIterate(cache->stbCache, NULL);
while (pIter) {
uint64_t *suid = NULL;
suid = taosHashGetKey(pIter, NULL);
if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) {
ctgDebug("stb removed from rent, suid:%"PRIx64, *suid);
}
pIter = taosHashIterate(cache->stbCache, pIter);
}
}
CTG_UNLOCK(CTG_WRITE, &cache->stbLock);
}
int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) {
uint64_t dbId = dbCache->dbId;
ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId);
atomic_store_8(&dbCache->deleted, 1);
ctgRemoveStbRent(pCtg, &dbCache->tbCache);
ctgFreeDbCache(dbCache);
CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare));
ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId);
if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) {
ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName);
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
CTG_CACHE_STAT_SUB(dbNum, 1);
ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) {
int32_t code = 0;
SCtgDBCache *dbCache = NULL;
ctgGetDBCache(pCtg, dbFName, &dbCache);
if (dbCache) {
// TODO OPEN IT
#if 0
if (dbCache->dbId == dbId) {
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
#else
if (0 == dbId) {
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
if (dbId && (dbCache->dbId == 0)) {
dbCache->dbId = dbId;
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
if (dbCache->dbId == dbId) {
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
#endif
CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName));
}
CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId));
ctgGetDBCache(pCtg, dbFName, &dbCache);
*pCache = dbCache;
return TSDB_CODE_SUCCESS;
}
int32_t ctgWriteDBVgInfoToCache(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) {
int32_t code = 0;
SDBVgInfo* dbInfo = *pDbInfo;
if (NULL == dbInfo->vgHash) {
return TSDB_CODE_SUCCESS;
}
if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) {
ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d",
dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
bool newAdded = false;
SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable};
SCtgDBCache *dbCache = NULL;
CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache));
if (NULL == dbCache) {
ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SDBVgInfo *vgInfo = NULL;
CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache));
if (dbCache->vgInfo) {
if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) {
ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion);
ctgWReleaseVgInfo(dbCache);
return TSDB_CODE_SUCCESS;
}
if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) {
ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable);
ctgWReleaseVgInfo(dbCache);
return TSDB_CODE_SUCCESS;
}
ctgFreeVgInfo(dbCache->vgInfo);
}
dbCache->vgInfo = dbInfo;
*pDbInfo = NULL;
ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId);
ctgWReleaseVgInfo(dbCache);
dbCache = NULL;
strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare));
CTG_RET(code);
}
int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) {
SCtgTbMetaCache *tbCache = &dbCache->tbCache;
CTG_LOCK(CTG_READ, &tbCache->metaLock);
if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) {
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId);
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
int8_t origType = 0;
uint64_t origSuid = 0;
bool isStb = meta->tableType == TSDB_SUPER_TABLE;
STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName));
if (orig) {
origType = orig->tableType;
if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) {
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
return TSDB_CODE_SUCCESS;
}
if (origType == TSDB_SUPER_TABLE) {
CTG_LOCK(CTG_WRITE, &tbCache->stbLock);
if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) {
ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid);
} else {
CTG_CACHE_STAT_SUB(stblNum, 1);
}
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid);
ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare);
origSuid = orig->suid;
}
}
if (isStb) {
CTG_LOCK(CTG_WRITE, &tbCache->stbLock);
}
if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) {
if (isStb) {
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
}
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
if (NULL == orig) {
CTG_CACHE_STAT_ADD(tblNum, 1);
}
ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
ctgdShowTableMeta(pCtg, tbName, meta);
if (!isStb) {
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
return TSDB_CODE_SUCCESS;
}
STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName));
if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) {
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
CTG_CACHE_STAT_ADD(stblNum, 1);
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion};
strcpy(metaRent.dbFName, dbFName);
strcpy(metaRent.stbName, tbName);
CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion)));
return TSDB_CODE_SUCCESS;
}
int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq) {
STableMetaOutput* pOutput = NULL;
int32_t code = 0;
CTG_ERR_RET(ctgCloneMetaOutput(pOut, &pOutput));
CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, pOutput, syncReq));
return TSDB_CODE_SUCCESS;
_return:
ctgFreeSTableMetaOutput(pOutput);
CTG_RET(code);
}
int32_t ctgActUpdateVg(SCtgMetaAction *action) {
int32_t code = 0;
SCtgUpdateVgMsg *msg = action->data;
CTG_ERR_JRET(ctgWriteDBVgInfoToCache(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo));
_return:
ctgFreeVgInfo(msg->dbInfo);
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgActRemoveDB(SCtgMetaAction *action) {
int32_t code = 0;
SCtgRemoveDBMsg *msg = action->data;
SCatalog* pCtg = msg->pCtg;
SCtgDBCache *dbCache = NULL;
ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache);
if (NULL == dbCache) {
goto _return;
}
if (dbCache->dbId != msg->dbId) {
ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId);
goto _return;
}
CTG_ERR_JRET(ctgRemoveDBFromCache(pCtg, dbCache, msg->dbFName));
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgActUpdateTb(SCtgMetaAction *action) {
int32_t code = 0;
SCtgUpdateTblMsg *msg = action->data;
SCatalog* pCtg = msg->pCtg;
STableMetaOutput* output = msg->output;
SCtgDBCache *dbCache = NULL;
if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) {
ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) {
ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache));
if (NULL == dbCache) {
ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) {
int32_t metaSize = CTG_META_SIZE(output->tbMeta);
CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize));
}
if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) {
CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta)));
}
_return:
if (output) {
taosMemoryFreeClear(output->tbMeta);
taosMemoryFreeClear(output);
}
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgActRemoveStb(SCtgMetaAction *action) {
int32_t code = 0;
SCtgRemoveStbMsg *msg = action->data;
SCatalog* pCtg = msg->pCtg;
SCtgDBCache *dbCache = NULL;
ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
if (NULL == dbCache) {
return TSDB_CODE_SUCCESS;
}
if (msg->dbId && (dbCache->dbId != msg->dbId)) {
ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid);
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock);
if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) {
ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
} else {
CTG_CACHE_STAT_SUB(stblNum, 1);
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) {
ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
} else {
CTG_CACHE_STAT_SUB(tblNum, 1);
}
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock);
ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare));
ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgActRemoveTb(SCtgMetaAction *action) {
int32_t code = 0;
SCtgRemoveTblMsg *msg = action->data;
SCatalog* pCtg = msg->pCtg;
SCtgDBCache *dbCache = NULL;
ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
if (NULL == dbCache) {
return TSDB_CODE_SUCCESS;
}
if (dbCache->dbId != msg->dbId) {
ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName);
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
} else {
CTG_CACHE_STAT_SUB(tblNum, 1);
}
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgActUpdateUser(SCtgMetaAction *action) {
int32_t code = 0;
SCtgUpdateUserMsg *msg = action->data;
SCatalog* pCtg = msg->pCtg;
if (NULL == pCtg->userCache) {
pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (NULL == pCtg->userCache) {
ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user));
if (NULL == pUser) {
SCtgUserAuth userAuth = {0};
userAuth.version = msg->userAuth.version;
userAuth.superUser = msg->userAuth.superAuth;
userAuth.createdDbs = msg->userAuth.createdDbs;
userAuth.readDbs = msg->userAuth.readDbs;
userAuth.writeDbs = msg->userAuth.writeDbs;
if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) {
ctgError("taosHashPut user %s to cache failed", msg->userAuth.user);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
taosMemoryFreeClear(msg);
return TSDB_CODE_SUCCESS;
}
pUser->version = msg->userAuth.version;
CTG_LOCK(CTG_WRITE, &pUser->lock);
taosHashCleanup(pUser->createdDbs);
pUser->createdDbs = msg->userAuth.createdDbs;
msg->userAuth.createdDbs = NULL;
taosHashCleanup(pUser->readDbs);
pUser->readDbs = msg->userAuth.readDbs;
msg->userAuth.readDbs = NULL;
taosHashCleanup(pUser->writeDbs);
pUser->writeDbs = msg->userAuth.writeDbs;
msg->userAuth.writeDbs = NULL;
CTG_UNLOCK(CTG_WRITE, &pUser->lock);
_return:
taosHashCleanup(msg->userAuth.createdDbs);
taosHashCleanup(msg->userAuth.readDbs);
taosHashCleanup(msg->userAuth.writeDbs);
taosMemoryFreeClear(msg);
CTG_RET(code);
}
void* ctgUpdateThreadFunc(void* param) {
setThreadName("catalog");
qInfo("catalog update thread started");
CTG_LOCK(CTG_READ, &gCtgMgmt.lock);
while (true) {
if (tsem_wait(&gCtgMgmt.queue.reqSem)) {
qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
}
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) {
tsem_post(&gCtgMgmt.queue.rspSem);
break;
}
SCtgMetaAction *action = NULL;
ctgPopAction(&action);
SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg;
ctgDebug("process [%s] action", gCtgAction[action->act].name);
(*gCtgAction[action->act].func)(action);
gCtgMgmt.queue.seqDone = action->seqId;
if (action->syncReq) {
tsem_post(&gCtgMgmt.queue.rspSem);
}
CTG_RUNTIME_STAT_ADD(qDoneNum, 1);
ctgdShowClusterCache(pCtg);
}
CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
qInfo("catalog update thread stopped");
return NULL;
}
int32_t ctgStartUpdateThread() {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
CTG_ERR_RET(terrno);
}
taosThreadAttrDestroy(&thAttr);
return TSDB_CODE_SUCCESS;
}
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "trpc.h"
#include "query.h"
#include "tname.h"
#include "catalogInt.h"
#include "systable.h"
#include "ctgRemote.h"
#include "tref.h"
int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target) {
int32_t code = 0;
switch (reqType) {
case TDMT_MND_QNODE_LIST: {
if (TSDB_CODE_SUCCESS != rspCode) {
qError("error rsp for qnode list, error:%s", tstrerror(rspCode));
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process qnode list rsp failed, error:%s", tstrerror(rspCode));
CTG_ERR_RET(code);
}
qDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out));
break;
}
case TDMT_MND_USE_DB: {
if (TSDB_CODE_SUCCESS != rspCode) {
qError("error rsp for use db, error:%s, dbFName:%s", tstrerror(rspCode), target);
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process use db rsp failed, error:%s, dbFName:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got db vgInfo from mnode, dbFName:%s", target);
break;
}
case TDMT_MND_GET_DB_CFG: {
if (TSDB_CODE_SUCCESS != rspCode) {
qError("error rsp for get db cfg, error:%s, db:%s", tstrerror(rspCode), target);
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get db cfg rsp failed, error:%s, db:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got db cfg from mnode, dbFName:%s", target);
break;
}
case TDMT_MND_GET_INDEX: {
if (TSDB_CODE_SUCCESS != rspCode) {
qError("error rsp for get index, error:%s, indexName:%s", tstrerror(rspCode), target);
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get index rsp failed, error:%s, indexName:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got index from mnode, indexName:%s", target);
break;
}
case TDMT_MND_RETRIEVE_FUNC: {
if (TSDB_CODE_SUCCESS != rspCode) {
qError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rspCode), target);
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get udf rsp failed, error:%s, funcName:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got udf from mnode, funcName:%s", target);
break;
}
case TDMT_MND_GET_USER_AUTH: {
if (TSDB_CODE_SUCCESS != rspCode) {
qError("error rsp for get user auth, error:%s, user:%s", tstrerror(rspCode), target);
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get user auth rsp failed, error:%s, user:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got user auth from mnode, user:%s", target);
break;
}
case TDMT_MND_TABLE_META: {
if (TSDB_CODE_SUCCESS != rspCode) {
if (CTG_TABLE_NOT_EXIST(rspCode)) {
SET_META_TYPE_NULL(((STableMetaOutput*)out)->metaType);
qDebug("stablemeta not exist in mnode, tbFName:%s", target);
return TSDB_CODE_SUCCESS;
}
qError("error rsp for stablemeta from mnode, error:%s, tbFName:%s", tstrerror(rspCode), target);
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process mnode stablemeta rsp failed, error:%s, tbFName:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got table meta from mnode, tbFName:%s", target);
break;
}
case TDMT_VND_TABLE_META: {
if (TSDB_CODE_SUCCESS != rspCode) {
if (CTG_TABLE_NOT_EXIST(rspCode)) {
SET_META_TYPE_NULL(((STableMetaOutput*)out)->metaType);
qDebug("tablemeta not exist in vnode, tbFName:%s", target);
return TSDB_CODE_SUCCESS;
}
qError("error rsp for table meta from vnode, code:%s, tbFName:%s", tstrerror(rspCode), target);
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process vnode tablemeta rsp failed, code:%s, tbFName:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got table meta from vnode, tbFName:%s", target);
break;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgHandleMsgCallback(void *param, const SDataBuf *pMsg, int32_t rspCode) {
SCtgTaskCallbackParam* cbParam = (SCtgTaskCallbackParam*)param;
int32_t code = 0;
CTG_API_ENTER();
SCtgJob* pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId);
if (NULL == pJob) {
qDebug("job refId %" PRIx64 " already dropped", cbParam->refId);
goto _return;
}
SCtgTask *pTask = taosArrayGet(pJob->pTasks, cbParam->taskId);
qDebug("QID:%" PRIx64 " task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1));
CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode));
_return:
if (pJob) {
taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId);
}
taosMemoryFree(param);
CTG_API_LEAVE(code);
}
int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsgSendInfo) {
int32_t code = 0;
SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == msgSendInfo) {
qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo));
CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SCtgTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam));
if (NULL == param) {
qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam));
CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
param->reqType = msgType;
param->queryId = pTask->pJob->queryId;
param->refId = pTask->pJob->refId;
param->taskId = pTask->taskId;
msgSendInfo->param = param;
msgSendInfo->fp = ctgHandleMsgCallback;
*pMsgSendInfo = msgSendInfo;
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFree(param);
taosMemoryFree(msgSendInfo);
CTG_RET(code);
}
int32_t ctgAsyncSendMsg(CTG_PARAMS, SCtgTask* pTask, int32_t msgType, void *msg, uint32_t msgSize) {
int32_t code = 0;
SMsgSendInfo *pMsgSendInfo = NULL;
CTG_ERR_JRET(ctgMakeMsgSendInfo(pTask, msgType, &pMsgSendInfo));
pMsgSendInfo->msgInfo.pData = msg;
pMsgSendInfo->msgInfo.len = msgSize;
pMsgSendInfo->msgInfo.handle = NULL;
pMsgSendInfo->msgType = msgType;
int64_t transporterId = 0;
code = asyncSendMsgToServer(pTrans, (SEpSet*)pMgmtEps, &transporterId, pMsgSendInfo);
if (code) {
ctgError("asyncSendMsgToSever failed, error: %s", tstrerror(code));
CTG_ERR_JRET(code);
}
ctgDebug("req msg sent, reqId:%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType));
return TSDB_CODE_SUCCESS;
_return:
if (pMsgSendInfo) {
taosMemoryFreeClear(pMsgSendInfo->param);
taosMemoryFreeClear(pMsgSendInfo);
}
CTG_RET(code);
}
int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask) {
char *msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_MND_QNODE_LIST;
ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pMgmtEps->inUse);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](NULL, &msg, 0, &msgLen);
if (code) {
ctgError("Build qnode list msg failed, error:%s", tstrerror(code));
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, NULL));
CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetDBVgInfoFromMnode(CTG_PARAMS, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask) {
char *msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_MND_USE_DB;
ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](input, &msg, 0, &msgLen);
if (code) {
ctgError("Build use db msg failed, code:%x, db:%s", code, input->db);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(SUseDbOutput));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, input->db));
CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, input->db));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetDBCfgFromMnode(CTG_PARAMS, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask) {
char *msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_MND_GET_DB_CFG;
ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)dbFName, &msg, 0, &msgLen);
if (code) {
ctgError("Build get db cfg msg failed, code:%x, db:%s", code, dbFName);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(SDbCfgInfo));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)dbFName));
CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_GET_DB_CFG,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)dbFName));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *out, SCtgTask* pTask) {
char *msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_MND_GET_INDEX;
ctgDebug("try to get index from mnode, indexName:%s", indexName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)indexName, &msg, 0, &msgLen);
if (code) {
ctgError("Build get index msg failed, code:%x, db:%s", code, indexName);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(SIndexInfo));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)indexName));
CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)indexName));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetUdfInfoFromMnode(CTG_PARAMS, const char *funcName, SFuncInfo *out, SCtgTask* pTask) {
char *msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_MND_RETRIEVE_FUNC;
ctgDebug("try to get udf info from mnode, funcName:%s", funcName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)funcName, &msg, 0, &msgLen);
if (code) {
ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(SFuncInfo));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)funcName));
CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)funcName));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetUserDbAuthFromMnode(CTG_PARAMS, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask) {
char *msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_MND_GET_USER_AUTH;
ctgDebug("try to get user auth from mnode, user:%s", user);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)user, &msg, 0, &msgLen);
if (code) {
ctgError("Build get user auth msg failed, code:%x, db:%s", code, user);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(SGetUserAuthRsp));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)user));
CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)user));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask) {
SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName};
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_MND_TABLE_META;
char tbFName[TSDB_TABLE_FNAME_LEN];
sprintf(tbFName, "%s.%s", dbFName, tbName);
ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen);
if (code) {
ctgError("Build mnode stablemeta msg failed, code:%x", code);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(STableMetaOutput));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName));
CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbMetaFromMnode(CTG_PARAMS, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
return ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), dbFName, (char *)pTableName->tname, out, pTask);
}
int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
int32_t reqType = TDMT_VND_TABLE_META;
char tbFName[TSDB_TABLE_FNAME_LEN];
sprintf(tbFName, "%s.%s", dbFName, pTableName->tname);
ctgDebug("try to get table meta from vnode, vgId:%d, tbFName:%s", vgroupInfo->vgId, tbFName);
SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)};
char *msg = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen);
if (code) {
ctgError("Build vnode tablemeta msg failed, code:%x, tbFName:%s", code, tbFName);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(STableMetaOutput));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName));
CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName));
return TSDB_CODE_SUCCESS;
}
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "trpc.h"
#include "query.h"
#include "tname.h"
#include "catalogInt.h"
#include "systable.h"
void ctgFreeSMetaData(SMetaData* pData) {
taosArrayDestroy(pData->pTableMeta);
pData->pTableMeta = NULL;
for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) {
SArray** pArray = taosArrayGet(pData->pDbVgroup, i);
taosArrayDestroy(*pArray);
}
taosArrayDestroy(pData->pDbVgroup);
pData->pDbVgroup = NULL;
taosArrayDestroy(pData->pTableHash);
pData->pTableHash = NULL;
taosArrayDestroy(pData->pUdfList);
pData->pUdfList = NULL;
for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) {
SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i);
taosArrayDestroy(pInfo->pRetensions);
}
taosArrayDestroy(pData->pDbCfg);
pData->pDbCfg = NULL;
taosArrayDestroy(pData->pIndex);
pData->pIndex = NULL;
taosArrayDestroy(pData->pUser);
pData->pUser = NULL;
taosArrayDestroy(pData->pQnodeList);
pData->pQnodeList = NULL;
}
void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) {
taosHashCleanup(userCache->createdDbs);
taosHashCleanup(userCache->readDbs);
taosHashCleanup(userCache->writeDbs);
}
void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
if (NULL == mgmt->slots) {
return;
}
for (int32_t i = 0; i < mgmt->slotNum; ++i) {
SCtgRentSlot *slot = &mgmt->slots[i];
if (slot->meta) {
taosArrayDestroy(slot->meta);
slot->meta = NULL;
}
}
taosMemoryFreeClear(mgmt->slots);
}
void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) {
CTG_LOCK(CTG_WRITE, &cache->stbLock);
if (cache->stbCache) {
int32_t stblNum = taosHashGetSize(cache->stbCache);
taosHashCleanup(cache->stbCache);
cache->stbCache = NULL;
CTG_CACHE_STAT_SUB(stblNum, stblNum);
}
CTG_UNLOCK(CTG_WRITE, &cache->stbLock);
CTG_LOCK(CTG_WRITE, &cache->metaLock);
if (cache->metaCache) {
int32_t tblNum = taosHashGetSize(cache->metaCache);
taosHashCleanup(cache->metaCache);
cache->metaCache = NULL;
CTG_CACHE_STAT_SUB(tblNum, tblNum);
}
CTG_UNLOCK(CTG_WRITE, &cache->metaLock);
}
void ctgFreeVgInfo(SDBVgInfo *vgInfo) {
if (NULL == vgInfo) {
return;
}
if (vgInfo->vgHash) {
taosHashCleanup(vgInfo->vgHash);
vgInfo->vgHash = NULL;
}
taosMemoryFreeClear(vgInfo);
}
void ctgFreeDbCache(SCtgDBCache *dbCache) {
if (NULL == dbCache) {
return;
}
CTG_LOCK(CTG_WRITE, &dbCache->vgLock);
ctgFreeVgInfo (dbCache->vgInfo);
CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock);
ctgFreeTbMetaCache(&dbCache->tbCache);
}
void ctgFreeHandle(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
if (pCtg->dbCache) {
int32_t dbNum = taosHashGetSize(pCtg->dbCache);
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
while (pIter) {
SCtgDBCache *dbCache = pIter;
atomic_store_8(&dbCache->deleted, 1);
ctgFreeDbCache(dbCache);
pIter = taosHashIterate(pCtg->dbCache, pIter);
}
taosHashCleanup(pCtg->dbCache);
CTG_CACHE_STAT_SUB(dbNum, dbNum);
}
if (pCtg->userCache) {
int32_t userNum = taosHashGetSize(pCtg->userCache);
void *pIter = taosHashIterate(pCtg->userCache, NULL);
while (pIter) {
SCtgUserAuth *userCache = pIter;
ctgFreeSCtgUserAuth(userCache);
pIter = taosHashIterate(pCtg->userCache, pIter);
}
taosHashCleanup(pCtg->userCache);
CTG_CACHE_STAT_SUB(userNum, userNum);
}
taosMemoryFree(pCtg);
}
void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) {
if (NULL == pOutput || NULL == pOutput->dbVgroup) {
return;
}
taosHashCleanup(pOutput->dbVgroup->vgHash);
taosMemoryFreeClear(pOutput->dbVgroup);
taosMemoryFree(pOutput);
}
void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) {
taosMemoryFreeClear(pCtx->target);
if (NULL == pCtx->out) {
return;
}
switch (pCtx->reqType) {
case TDMT_MND_GET_DB_CFG: {
SDbCfgInfo* pOut = (SDbCfgInfo*)pCtx->out;
taosArrayDestroy(pOut->pRetensions);
taosMemoryFreeClear(pCtx->out);
break;
}
case TDMT_MND_USE_DB:{
SUseDbOutput* pOut = (SUseDbOutput*)pCtx->out;
ctgFreeSUseDbOutput(pOut);
pCtx->out = NULL;
break;
}
case TDMT_MND_GET_INDEX: {
SIndexInfo* pOut = (SIndexInfo*)pCtx->out;
taosMemoryFreeClear(pCtx->out);
break;
}
case TDMT_MND_QNODE_LIST: {
SArray* pOut = (SArray*)pCtx->out;
taosArrayDestroy(pOut);
pCtx->out = NULL;
break;
}
case TDMT_VND_TABLE_META:
case TDMT_MND_TABLE_META: {
STableMetaOutput* pOut = (STableMetaOutput*)pCtx->out;
taosMemoryFree(pOut->tbMeta);
taosMemoryFreeClear(pCtx->out);
break;
}
case TDMT_MND_RETRIEVE_FUNC: {
SFuncInfo* pOut = (SFuncInfo*)pCtx->out;
taosMemoryFree(pOut->pCode);
taosMemoryFree(pOut->pComment);
taosMemoryFreeClear(pCtx->out);
break;
}
case TDMT_MND_GET_USER_AUTH: {
SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pCtx->out;
taosHashCleanup(pOut->createdDbs);
taosHashCleanup(pOut->readDbs);
taosHashCleanup(pOut->writeDbs);
taosMemoryFreeClear(pCtx->out);
break;
}
default:
qError("invalid reqType %d", pCtx->reqType);
break;
}
}
void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) {
if (NULL == pOutput) {
return;
}
taosMemoryFree(pOutput->tbMeta);
taosMemoryFree(pOutput);
}
void ctgResetTbMetaTask(SCtgTask* pTask) {
SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx;
memset(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo));
taskCtx->flag = CTG_FLAG_UNKNOWN_STB;
if (pTask->msgCtx.lastOut) {
ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut);
pTask->msgCtx.lastOut = NULL;
}
if (pTask->msgCtx.out) {
ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.out);
pTask->msgCtx.out = NULL;
}
taosMemoryFreeClear(pTask->msgCtx.target);
taosMemoryFreeClear(pTask->res);
}
void ctgFreeTask(SCtgTask* pTask) {
ctgFreeMsgCtx(&pTask->msgCtx);
switch (pTask->type) {
case CTG_TASK_GET_QNODE: {
taosArrayDestroy((SArray*)pTask->res);
pTask->res = NULL;
break;
}
case CTG_TASK_GET_TB_META: {
SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx;
taosMemoryFreeClear(taskCtx->pName);
if (pTask->msgCtx.lastOut) {
ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut);
pTask->msgCtx.lastOut = NULL;
}
taosMemoryFreeClear(pTask->res);
break;
}
case CTG_TASK_GET_DB_VGROUP: {
taosArrayDestroy((SArray*)pTask->res);
pTask->res = NULL;
break;
}
case CTG_TASK_GET_DB_CFG: {
if (pTask->res) {
taosArrayDestroy(((SDbCfgInfo*)pTask->res)->pRetensions);
taosMemoryFreeClear(pTask->res);
}
break;
}
case CTG_TASK_GET_TB_HASH: {
SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx;
taosMemoryFreeClear(taskCtx->pName);
taosMemoryFreeClear(pTask->res);
break;
}
case CTG_TASK_GET_INDEX: {
taosMemoryFreeClear(pTask->taskCtx);
taosMemoryFreeClear(pTask->res);
break;
}
case CTG_TASK_GET_UDF: {
taosMemoryFreeClear(pTask->taskCtx);
taosMemoryFreeClear(pTask->res);
break;
}
case CTG_TASK_GET_USER: {
taosMemoryFreeClear(pTask->taskCtx);
taosMemoryFreeClear(pTask->res);
break;
}
default:
qError("invalid task type %d", pTask->type);
break;
}
}
void ctgFreeTasks(SArray* pArray) {
if (NULL == pArray) {
return;
}
int32_t num = taosArrayGetSize(pArray);
for (int32_t i = 0; i < num; ++i) {
SCtgTask* pTask = taosArrayGet(pArray, i);
ctgFreeTask(pTask);
}
taosArrayDestroy(pArray);
}
void ctgFreeJob(void* job) {
if (NULL == job) {
return;
}
SCtgJob* pJob = (SCtgJob*)job;
int64_t rid = pJob->refId;
uint64_t qid = pJob->queryId;
ctgFreeTasks(pJob->pTasks);
ctgFreeSMetaData(&pJob->jobRes);
taosMemoryFree(job);
qDebug("QID:%" PRIx64 ", job %" PRIx64 " freed", qid, rid);
}
int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target) {
ctgFreeMsgCtx(pCtx);
pCtx->reqType = reqType;
pCtx->out = out;
if (target) {
pCtx->target = strdup(target);
if (NULL == pCtx->target) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} else {
pCtx->target = NULL;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) {
switch (hashMethod) {
default:
*fp = MurmurHash3_32;
break;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) {
SHashObj *vgroupHash = NULL;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
int32_t code = 0;
int32_t vgNum = taosHashGetSize(vgHash);
vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo));
if (NULL == vgList) {
ctgError("taosArrayInit failed, num:%d", vgNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
void *pIter = taosHashIterate(vgHash, NULL);
while (pIter) {
vgInfo = pIter;
if (NULL == taosArrayPush(vgList, vgInfo)) {
ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId);
taosHashCancelIterate(vgHash, pIter);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
pIter = taosHashIterate(vgHash, pIter);
vgInfo = NULL;
}
*pList = vgList;
ctgDebug("Got vgList from cache, vgNum:%d", vgNum);
return TSDB_CODE_SUCCESS;
_return:
if (vgList) {
taosArrayDestroy(vgList);
}
CTG_RET(code);
}
int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) {
int32_t code = 0;
int32_t vgNum = taosHashGetSize(dbInfo->vgHash);
char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
if (vgNum <= 0) {
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
}
tableNameHashFp fp = NULL;
SVgroupInfo *vgInfo = NULL;
CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp));
char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName));
void *pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) {
vgInfo = pIter;
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
taosHashCancelIterate(dbInfo->vgHash, pIter);
break;
}
pIter = taosHashIterate(dbInfo->vgHash, pIter);
vgInfo = NULL;
}
if (NULL == vgInfo) {
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
*pVgroup = *vgInfo;
CTG_RET(code);
}
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) {
if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) {
return -1;
} else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) {
return 1;
} else {
return 0;
}
}
int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) {
if (*(int64_t *)key1 < ((SDbVgVersion*)key2)->dbId) {
return -1;
} else if (*(int64_t *)key1 > ((SDbVgVersion*)key2)->dbId) {
return 1;
} else {
return 0;
}
}
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) {
if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) {
return -1;
} else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) {
return 1;
} else {
return 0;
}
}
int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) {
if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) {
return -1;
} else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) {
return 1;
} else {
return 0;
}
}
int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
*dst = taosMemoryMalloc(sizeof(SDBVgInfo));
if (NULL == *dst) {
qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(*dst, src, sizeof(SDBVgInfo));
size_t hashSize = taosHashGetSize(src->vgHash);
(*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
if (NULL == (*dst)->vgHash) {
qError("taosHashInit %d failed", (int32_t)hashSize);
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
int32_t *vgId = NULL;
void *pIter = taosHashIterate(src->vgHash, NULL);
while (pIter) {
vgId = taosHashGetKey(pIter, NULL);
if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) {
qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize);
taosHashCancelIterate(src->vgHash, pIter);
taosHashCleanup((*dst)->vgHash);
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
pIter = taosHashIterate(src->vgHash, pIter);
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) {
*pOutput = taosMemoryMalloc(sizeof(STableMetaOutput));
if (NULL == *pOutput) {
qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(*pOutput, output, sizeof(STableMetaOutput));
if (output->tbMeta) {
int32_t metaSize = CTG_META_SIZE(output->tbMeta);
(*pOutput)->tbMeta = taosMemoryMalloc(metaSize);
if (NULL == (*pOutput)->tbMeta) {
qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
taosMemoryFreeClear(*pOutput);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize);
}
return TSDB_CODE_SUCCESS;
}
...@@ -40,10 +40,8 @@ ...@@ -40,10 +40,8 @@
namespace { namespace {
extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta,
bool *inCache, int32_t flag, uint64_t *dbId);
extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); extern "C" int32_t ctgActUpdateTb(SCtgMetaAction *action);
extern "C" int32_t ctgdEnableDebug(char *option); extern "C" int32_t ctgdEnableDebug(char *option);
extern "C" int32_t ctgdGetStatNum(char *option, void *res); extern "C" int32_t ctgdGetStatNum(char *option, void *res);
...@@ -52,7 +50,7 @@ void ctgTestSetRspCTableMeta(); ...@@ -52,7 +50,7 @@ void ctgTestSetRspCTableMeta();
void ctgTestSetRspSTableMeta(); void ctgTestSetRspSTableMeta();
void ctgTestSetRspMultiSTableMeta(); void ctgTestSetRspMultiSTableMeta();
extern "C" SCatalogMgmt gCtgMgmt; //extern "C" SCatalogMgmt gCtgMgmt;
enum { enum {
CTGT_RSP_VGINFO = 1, CTGT_RSP_VGINFO = 1,
...@@ -859,8 +857,12 @@ void *ctgTestGetCtableMetaThread(void *param) { ...@@ -859,8 +857,12 @@ void *ctgTestGetCtableMetaThread(void *param) {
strcpy(cn.dbname, "db1"); strcpy(cn.dbname, "db1");
strcpy(cn.tname, ctgTestCTablename); strcpy(cn.tname, ctgTestCTablename);
SCtgTbMetaCtx ctx = {0};
ctx.pName = &cn;
ctx.flag = CTG_FLAG_UNKNOWN_STB;
while (!ctgTestStop) { while (!ctgTestStop) {
code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL); code = ctgReadTbMetaFromCache(pCtg, &ctx, &tbMeta);
if (code || !inCache) { if (code || !inCache) {
assert(0); assert(0);
} }
...@@ -899,7 +901,7 @@ void *ctgTestSetCtableMetaThread(void *param) { ...@@ -899,7 +901,7 @@ void *ctgTestSetCtableMetaThread(void *param) {
msg->output = output; msg->output = output;
action.data = msg; action.data = msg;
code = ctgActUpdateTbl(&action); code = ctgActUpdateTb(&action);
if (code) { if (code) {
assert(0); assert(0);
} }
......
...@@ -13,7 +13,6 @@ ...@@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <executorimpl.h>
#include "filter.h" #include "filter.h"
#include "function.h" #include "function.h"
#include "functionMgt.h" #include "functionMgt.h"
......
...@@ -1181,6 +1181,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* ...@@ -1181,6 +1181,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols); pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols);
pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API
// pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo);
if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) {
......
...@@ -66,22 +66,18 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { ...@@ -66,22 +66,18 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
} }
static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) {
SFuncInfo* pInfo = NULL; SFuncInfo funcInfo = {0};
int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo); int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &funcInfo);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
if (NULL == pInfo) {
snprintf(pParam->pErrBuf, pParam->errBufLen, "Invalid function name: %s", pFunc->functionName);
return TSDB_CODE_FUNC_INVALID_FUNTION;
}
pFunc->funcType = FUNCTION_TYPE_UDF; pFunc->funcType = FUNCTION_TYPE_UDF;
pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == pInfo->funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID; pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == funcInfo.funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID;
pFunc->node.resType.type = pInfo->outputType; pFunc->node.resType.type = funcInfo.outputType;
pFunc->node.resType.bytes = pInfo->outputLen; pFunc->node.resType.bytes = funcInfo.outputLen;
pFunc->udfBufSize = pInfo->bufSize; pFunc->udfBufSize = funcInfo.bufSize;
tFreeSFuncInfo(pInfo); tFreeSFuncInfo(&funcInfo);
taosMemoryFree(pInfo);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
...@@ -72,12 +72,20 @@ static int32_t udfSpawnUdfd(SUdfdData* pData) { ...@@ -72,12 +72,20 @@ static int32_t udfSpawnUdfd(SUdfdData* pData) {
char path[PATH_MAX] = {0}; char path[PATH_MAX] = {0};
if (tsProcPath == NULL) { if (tsProcPath == NULL) {
path[0] = '.'; path[0] = '.';
#ifdef WINDOWS
GetModuleFileName(NULL, path, PATH_MAX);
taosDirName(path);
#endif
} else { } else {
strncpy(path, tsProcPath, strlen(tsProcPath)); strncpy(path, tsProcPath, strlen(tsProcPath));
taosDirName(path); taosDirName(path);
} }
#ifdef WINDOWS #ifdef WINDOWS
if (strlen(path)==0) {
strcat(path, "udfd.exe"); strcat(path, "udfd.exe");
} else {
strcat(path, "\\udfd.exe");
}
#else #else
strcat(path, "/udfd"); strcat(path, "/udfd");
#endif #endif
......
...@@ -204,7 +204,7 @@ TEST_F(ParserInitialATest, alterTable) { ...@@ -204,7 +204,7 @@ TEST_F(ParserInitialATest, alterTable) {
} }
}; };
auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, const uint8_t* pNewVal, uint32_t bytes) { auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, uint8_t* pNewVal, uint32_t bytes) {
memset(&expect, 0, sizeof(SVAlterTbReq)); memset(&expect, 0, sizeof(SVAlterTbReq));
expect.tbName = strdup(pTbname); expect.tbName = strdup(pTbname);
expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL;
...@@ -215,7 +215,7 @@ TEST_F(ParserInitialATest, alterTable) { ...@@ -215,7 +215,7 @@ TEST_F(ParserInitialATest, alterTable) {
expect.pTagVal = pNewVal; expect.pTagVal = pNewVal;
}; };
auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, const char* pComment = nullptr) { auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, char* pComment = nullptr) {
memset(&expect, 0, sizeof(SVAlterTbReq)); memset(&expect, 0, sizeof(SVAlterTbReq));
expect.tbName = strdup(pTbname); expect.tbName = strdup(pTbname);
expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS; expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS;
...@@ -240,7 +240,7 @@ TEST_F(ParserInitialATest, alterTable) { ...@@ -240,7 +240,7 @@ TEST_F(ParserInitialATest, alterTable) {
void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead));
SVAlterTbReq req = {0}; SVAlterTbReq req = {0};
SDecoder coder = {0}; SDecoder coder = {0};
tDecoderInit(&coder, (const uint8_t*)pBuf, pVgData->size); tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size);
ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS);
ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName)); ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName));
...@@ -274,7 +274,7 @@ TEST_F(ParserInitialATest, alterTable) { ...@@ -274,7 +274,7 @@ TEST_F(ParserInitialATest, alterTable) {
setAlterOptionsFunc("t1", 10, nullptr); setAlterOptionsFunc("t1", 10, nullptr);
run("ALTER TABLE t1 TTL 10"); run("ALTER TABLE t1 TTL 10");
setAlterOptionsFunc("t1", -1, "test"); setAlterOptionsFunc("t1", -1, (char*)"test");
run("ALTER TABLE t1 COMMENT 'test'"); run("ALTER TABLE t1 COMMENT 'test'");
setAlterColFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT); setAlterColFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT);
...@@ -290,7 +290,7 @@ TEST_F(ParserInitialATest, alterTable) { ...@@ -290,7 +290,7 @@ TEST_F(ParserInitialATest, alterTable) {
run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); run("ALTER TABLE t1 RENAME COLUMN c1 cc1");
int32_t val = 10; int32_t val = 10;
setAlterTagFunc("st1s1", "tag1", (const uint8_t*)&val, sizeof(val)); setAlterTagFunc("st1s1", "tag1", (uint8_t*)&val, sizeof(val));
run("ALTER TABLE st1s1 SET TAG tag1=10"); run("ALTER TABLE st1s1 SET TAG tag1=10");
// todo // todo
......
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
*/ */
#include "planTestUtil.h" #include "planTestUtil.h"
#include <getopt.h>
#include <algorithm> #include <algorithm>
#include <array> #include <array>
......
...@@ -528,20 +528,18 @@ int32_t qwDropTask(QW_FPARAMS_DEF) { ...@@ -528,20 +528,18 @@ int32_t qwDropTask(QW_FPARAMS_DEF) {
} }
int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
qTaskInfo_t *taskHandle = &ctx->taskHandle; qTaskInfo_t taskHandle = ctx->taskHandle;
if (TASK_TYPE_TEMP == ctx->taskType) { if (TASK_TYPE_TEMP == ctx->taskType && taskHandle) {
if (ctx->explain) { if (ctx->explain) {
SExplainExecInfo *execInfo = NULL; SExplainExecInfo *execInfo = NULL;
int32_t resNum = 0; int32_t resNum = 0;
QW_ERR_RET(qGetExplainExecInfo(ctx->taskHandle, &resNum, &execInfo)); QW_ERR_RET(qGetExplainExecInfo(taskHandle, &resNum, &execInfo));
SRpcHandleInfo connInfo = ctx->ctrlConnInfo; SRpcHandleInfo connInfo = ctx->ctrlConnInfo;
connInfo.ahandle = NULL; connInfo.ahandle = NULL;
QW_ERR_RET(qwBuildAndSendExplainRsp(&connInfo, execInfo, resNum)); QW_ERR_RET(qwBuildAndSendExplainRsp(&connInfo, execInfo, resNum));
} }
qwFreeTaskHandle(QW_FPARAMS(), taskHandle);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -554,17 +552,22 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { ...@@ -554,17 +552,22 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) {
uint64_t useconds = 0; uint64_t useconds = 0;
int32_t i = 0; int32_t i = 0;
int32_t execNum = 0; int32_t execNum = 0;
qTaskInfo_t *taskHandle = &ctx->taskHandle; qTaskInfo_t taskHandle = ctx->taskHandle;
DataSinkHandle sinkHandle = ctx->sinkHandle; DataSinkHandle sinkHandle = ctx->sinkHandle;
while (true) { while (true) {
QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); QW_TASK_DLOG("start to execTask, loopIdx:%d", i++);
code = qExecTask(*taskHandle, &pRes, &useconds); pRes = NULL;
// if *taskHandle is NULL, it's killed right now
if (taskHandle) {
code = qExecTask(taskHandle, &pRes, &useconds);
if (code) { if (code) {
QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code));
QW_ERR_RET(code); QW_ERR_RET(code);
} }
}
++execNum; ++execNum;
......
...@@ -132,7 +132,7 @@ typedef struct SSchLevel { ...@@ -132,7 +132,7 @@ typedef struct SSchLevel {
int32_t taskSucceed; int32_t taskSucceed;
int32_t taskNum; int32_t taskNum;
int32_t taskLaunchedNum; int32_t taskLaunchedNum;
SHashObj *flowCtrl; // key is ep, element is SSchFlowControl int32_t taskDoneNum;
SArray *subTasks; // Element is SQueryTask SArray *subTasks; // Element is SQueryTask
} SSchLevel; } SSchLevel;
...@@ -175,11 +175,13 @@ typedef struct SSchJob { ...@@ -175,11 +175,13 @@ typedef struct SSchJob {
SArray *levels; // starting from 0. SArray<SSchLevel> SArray *levels; // starting from 0. SArray<SSchLevel>
SNodeList *subPlans; // subplan pointer copied from DAG, no need to free it in scheduler SNodeList *subPlans; // subplan pointer copied from DAG, no need to free it in scheduler
SArray *dataSrcTasks; // SArray<SQueryTask*>
int32_t levelIdx; int32_t levelIdx;
SEpSet dataSrcEps; SEpSet dataSrcEps;
SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask* SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask*
SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask*
SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask*
SHashObj *flowCtrl; // key is ep, element is SSchFlowControl
SExplainCtx *explainCtx; SExplainCtx *explainCtx;
int8_t status; int8_t status;
...@@ -200,7 +202,7 @@ typedef struct SSchJob { ...@@ -200,7 +202,7 @@ typedef struct SSchJob {
extern SSchedulerMgmt schMgmt; extern SSchedulerMgmt schMgmt;
#define SCH_TASK_READY_TO_LUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) #define SCH_TASK_READY_FOR_LAUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children))
#define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1)
#define SCH_SET_TASK_LASTMSG_TYPE(_task, _type) do { if(_task) { atomic_store_32(&(_task)->lastMsgType, _type); } } while (0) #define SCH_SET_TASK_LASTMSG_TYPE(_task, _type) do { if(_task) { atomic_store_32(&(_task)->lastMsgType, _type); } } while (0)
...@@ -223,7 +225,7 @@ extern SSchedulerMgmt schMgmt; ...@@ -223,7 +225,7 @@ extern SSchedulerMgmt schMgmt;
#define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true
#define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) #define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl)
#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEAF_TASK(_job, _task) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) #define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level))
#define SCH_SET_JOB_TYPE(_job, type) (_job)->attr.queryJob = ((type) != SUBPLAN_TYPE_MODIFY) #define SCH_SET_JOB_TYPE(_job, type) (_job)->attr.queryJob = ((type) != SUBPLAN_TYPE_MODIFY)
#define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob)
...@@ -261,7 +263,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task); ...@@ -261,7 +263,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task);
int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType);
SSchJob *schAcquireJob(int64_t refId); SSchJob *schAcquireJob(int64_t refId);
int32_t schReleaseJob(int64_t refId); int32_t schReleaseJob(int64_t refId);
void schFreeFlowCtrl(SSchLevel *pLevel); void schFreeFlowCtrl(SSchJob *pJob);
int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel);
int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask);
int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough);
......
...@@ -19,13 +19,13 @@ ...@@ -19,13 +19,13 @@
#include "catalog.h" #include "catalog.h"
#include "tref.h" #include "tref.h"
void schFreeFlowCtrl(SSchLevel *pLevel) { void schFreeFlowCtrl(SSchJob *pJob) {
if (NULL == pLevel->flowCtrl) { if (NULL == pJob->flowCtrl) {
return; return;
} }
SSchFlowControl *ctrl = NULL; SSchFlowControl *ctrl = NULL;
void *pIter = taosHashIterate(pLevel->flowCtrl, NULL); void *pIter = taosHashIterate(pJob->flowCtrl, NULL);
while (pIter) { while (pIter) {
ctrl = (SSchFlowControl *)pIter; ctrl = (SSchFlowControl *)pIter;
...@@ -33,11 +33,11 @@ void schFreeFlowCtrl(SSchLevel *pLevel) { ...@@ -33,11 +33,11 @@ void schFreeFlowCtrl(SSchLevel *pLevel) {
taosArrayDestroy(ctrl->taskList); taosArrayDestroy(ctrl->taskList);
} }
pIter = taosHashIterate(pLevel->flowCtrl, pIter); pIter = taosHashIterate(pJob->flowCtrl, pIter);
} }
taosHashCleanup(pLevel->flowCtrl); taosHashCleanup(pJob->flowCtrl);
pLevel->flowCtrl = NULL; pJob->flowCtrl = NULL;
} }
int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
...@@ -47,9 +47,9 @@ int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { ...@@ -47,9 +47,9 @@ int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
} }
int32_t sum = 0; int32_t sum = 0;
int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks);
for (int32_t i = 0; i < pLevel->taskNum; ++i) { for (int32_t i = 0; i < taskNum; ++i) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, i); SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i);
sum += pTask->plan->execNodeStat.tableNum; sum += pTask->plan->execNodeStat.tableNum;
} }
...@@ -59,9 +59,9 @@ int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { ...@@ -59,9 +59,9 @@ int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pLevel->flowCtrl = taosHashInit(pLevel->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); pJob->flowCtrl = taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (NULL == pLevel->flowCtrl) { if (NULL == pJob->flowCtrl) {
SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pLevel->taskNum); SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pJob->taskNum);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
...@@ -78,7 +78,7 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) { ...@@ -78,7 +78,7 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) {
int32_t code = 0; int32_t code = 0;
SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp)); ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp));
if (NULL == ctrl) { if (NULL == ctrl) {
SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
...@@ -110,11 +110,11 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { ...@@ -110,11 +110,11 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) {
SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
do { do {
ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp)); ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp));
if (NULL == ctrl) { if (NULL == ctrl) {
SSchFlowControl nctrl = {.tableNumSum = pTask->plan->execNodeStat.tableNum, .execTaskNum = 1}; SSchFlowControl nctrl = {.tableNumSum = pTask->plan->execNodeStat.tableNum, .execTaskNum = 1};
code = taosHashPut(pLevel->flowCtrl, ep, sizeof(SEp), &nctrl, sizeof(nctrl)); code = taosHashPut(pJob->flowCtrl, ep, sizeof(SEp), &nctrl, sizeof(nctrl));
if (code) { if (code) {
if (HASH_NODE_EXIST(code)) { if (HASH_NODE_EXIST(code)) {
continue; continue;
...@@ -273,10 +273,9 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { ...@@ -273,10 +273,9 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask)); SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask));
SSchLevel *pLevel = pTask->level;
SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
SSchFlowControl *ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp)); SSchFlowControl *ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp));
if (NULL == ctrl) { if (NULL == ctrl) {
SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
......
...@@ -391,6 +391,8 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { ...@@ -391,6 +391,8 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
SCH_TASK_ELOG("taosArrayPush childTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_TASK_ELOG("taosArrayPush childTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SCH_TASK_DLOG("children info, the %d child TID %" PRIx64, n, (*childTask)->taskId);
} }
if (parentNum > 0) { if (parentNum > 0) {
...@@ -423,6 +425,8 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { ...@@ -423,6 +425,8 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SCH_TASK_DLOG("parents info, the %d parent TID %" PRIx64, n, (*parentTask)->taskId);
} }
SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum);
...@@ -464,6 +468,17 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad ...@@ -464,6 +468,17 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schRecordQueryDataSrc(SSchJob *pJob, SSchTask *pTask) {
if (!SCH_IS_DATA_SRC_QRY_TASK(pTask)) {
return TSDB_CODE_SUCCESS;
}
taosArrayPush(pJob->dataSrcTasks, &pTask);
return TSDB_CODE_SUCCESS;
}
int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
int32_t code = 0; int32_t code = 0;
pJob->queryId = pDag->queryId; pJob->queryId = pDag->queryId;
...@@ -473,6 +488,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { ...@@ -473,6 +488,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
pJob->dataSrcTasks = taosArrayInit(pDag->numOfSubplans, POINTER_BYTES);
if (NULL == pJob->dataSrcTasks) {
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans);
if (levelNum <= 0) { if (levelNum <= 0) {
SCH_JOB_ELOG("invalid level num:%d", levelNum); SCH_JOB_ELOG("invalid level num:%d", levelNum);
...@@ -551,6 +571,8 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { ...@@ -551,6 +571,8 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SCH_ERR_JRET(schRecordQueryDataSrc(pJob, p));
if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) {
SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
...@@ -629,6 +651,17 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { ...@@ -629,6 +651,17 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) {
int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId));
if (code) {
SCH_TASK_ELOG("task failed to rm from execTask list, code:%x", code);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
return TSDB_CODE_SUCCESS;
}
int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) {
int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES);
if (0 != code) { if (0 != code) {
...@@ -774,6 +807,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo ...@@ -774,6 +807,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
SCH_ERR_RET(schRemoveTaskFromExecList(pJob, pTask));
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START);
if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask)); SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask));
SCH_ERR_RET(schLaunchTasksInFlowCtrlList(pJob, pTask)); SCH_ERR_RET(schLaunchTasksInFlowCtrlList(pJob, pTask));
...@@ -947,6 +983,32 @@ _return: ...@@ -947,6 +983,32 @@ _return:
SCH_RET(schProcessOnJobFailure(pJob, errCode)); SCH_RET(schProcessOnJobFailure(pJob, errCode));
} }
int32_t schLaunchNextLevelTasks(SSchJob *pJob, SSchTask *pTask) {
if (!SCH_IS_QUERY_JOB(pJob)) {
return TSDB_CODE_SUCCESS;
}
SSchLevel *pLevel = pTask->level;
int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1);
if (doneNum == pLevel->taskNum) {
pJob->levelIdx--;
pLevel = taosArrayGet(pJob->levels, pJob->levelIdx);
for (int32_t i = 0; i < pLevel->taskNum; ++i) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, i);
if (pTask->children && taosArrayGetSize(pTask->children) > 0) {
continue;
}
SCH_ERR_RET(schLaunchTask(pJob, pTask));
}
}
return TSDB_CODE_SUCCESS;
}
// Note: no more task error processing, handled in function internal // Note: no more task error processing, handled in function internal
int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
bool moved = false; bool moved = false;
...@@ -1015,11 +1077,13 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { ...@@ -1015,11 +1077,13 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source); qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source);
SCH_UNLOCK(SCH_WRITE, &par->lock); SCH_UNLOCK(SCH_WRITE, &par->lock);
if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { if (SCH_TASK_READY_FOR_LAUNCH(readyNum, par)) {
SCH_ERR_RET(schLaunchTaskImpl(pJob, par)); SCH_ERR_RET(schLaunchTask(pJob, par));
} }
} }
SCH_ERR_RET(schLaunchNextLevelTasks(pJob, pTask));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
...@@ -2400,8 +2464,6 @@ void schFreeJobImpl(void *job) { ...@@ -2400,8 +2464,6 @@ void schFreeJobImpl(void *job) {
for (int32_t i = 0; i < numOfLevels; ++i) { for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i); SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
schFreeFlowCtrl(pLevel);
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for (int32_t j = 0; j < numOfTasks; ++j) { for (int32_t j = 0; j < numOfTasks; ++j) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
...@@ -2411,12 +2473,15 @@ void schFreeJobImpl(void *job) { ...@@ -2411,12 +2473,15 @@ void schFreeJobImpl(void *job) {
taosArrayDestroy(pLevel->subTasks); taosArrayDestroy(pLevel->subTasks);
} }
schFreeFlowCtrl(pJob);
taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->execTasks);
taosHashCleanup(pJob->failTasks); taosHashCleanup(pJob->failTasks);
taosHashCleanup(pJob->succTasks); taosHashCleanup(pJob->succTasks);
taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->levels);
taosArrayDestroy(pJob->nodeList); taosArrayDestroy(pJob->nodeList);
taosArrayDestroy(pJob->dataSrcTasks);
qExplainFreeCtx(pJob->explainCtx); qExplainFreeCtx(pJob->explainCtx);
......
...@@ -19,18 +19,24 @@ ...@@ -19,18 +19,24 @@
#define DEFAULT_FALSE_POSITIVE 0.01 #define DEFAULT_FALSE_POSITIVE 0.01
#define DEFAULT_BUCKET_SIZE 1024 #define DEFAULT_BUCKET_SIZE 1024
#define ROWS_PER_MILLISECOND 1 #define ROWS_PER_MILLISECOND 1
#define MAX_NUM_SCALABLE_BF 120 #define MAX_NUM_SCALABLE_BF 100000
#define MIN_NUM_SCALABLE_BF 10 #define MIN_NUM_SCALABLE_BF 10
#define DEFAULT_PREADD_BUCKET 1 #define DEFAULT_PREADD_BUCKET 1
#define MAX_INTERVAL MILLISECOND_PER_MINUTE #define MAX_INTERVAL MILLISECOND_PER_MINUTE
#define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10) #define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10)
#define DEFAULT_EXPECTED_ENTRIES 10000
static int64_t adjustExpEntries(int64_t entries) {
return TMIN(DEFAULT_EXPECTED_ENTRIES, entries);
}
static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) {
if (pInfo->numSBFs < count) { if (pInfo->numSBFs < count) {
count = pInfo->numSBFs; count = pInfo->numSBFs;
} }
for (uint64_t i = 0; i < count; ++i) { for (uint64_t i = 0; i < count; ++i) {
SScalableBf *tsSBF = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, DEFAULT_FALSE_POSITIVE); int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
SScalableBf *tsSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE);
taosArrayPush(pInfo->pTsSBFs, &tsSBF); taosArrayPush(pInfo->pTsSBFs, &tsSBF);
} }
} }
...@@ -38,9 +44,9 @@ static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { ...@@ -38,9 +44,9 @@ static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) {
static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) {
if (count < pInfo->numSBFs - 1) { if (count < pInfo->numSBFs - 1) {
for (uint64_t i = 0; i < count; ++i) { for (uint64_t i = 0; i < count; ++i) {
SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, i); SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0);
tScalableBfDestroy(pTsSBFs); tScalableBfDestroy(pTsSBFs);
taosArrayRemove(pInfo->pTsSBFs, i); taosArrayRemove(pInfo->pTsSBFs, 0);
} }
} else { } else {
taosArrayClearP(pInfo->pTsSBFs, (FDelete)tScalableBfDestroy); taosArrayClearP(pInfo->pTsSBFs, (FDelete)tScalableBfDestroy);
...@@ -66,7 +72,7 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) { ...@@ -66,7 +72,7 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) {
return val; return val;
} }
static int64_t adjustWatermark(int64_t interval, int32_t watermark) { static int64_t adjustWatermark(int64_t interval, int64_t watermark) {
if (watermark <= 0 || watermark > MAX_NUM_SCALABLE_BF * interval) { if (watermark <= 0 || watermark > MAX_NUM_SCALABLE_BF * interval) {
watermark = MAX_NUM_SCALABLE_BF * interval; watermark = MAX_NUM_SCALABLE_BF * interval;
} else if (watermark < MIN_NUM_SCALABLE_BF * interval) { } else if (watermark < MIN_NUM_SCALABLE_BF * interval) {
...@@ -130,7 +136,8 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { ...@@ -130,7 +136,8 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) {
} }
SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index);
if (res == NULL) { if (res == NULL) {
res = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, DEFAULT_FALSE_POSITIVE); int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
res = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE);
taosArrayPush(pInfo->pTsSBFs, &res); taosArrayPush(pInfo->pTsSBFs, &res);
} }
return res; return res;
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
#include "ttime.h" #include "ttime.h"
using namespace std; using namespace std;
#define MAX_NUM_SCALABLE_BF 100000
TEST(TD_STREAM_UPDATE_TEST, update) { TEST(TD_STREAM_UPDATE_TEST, update) {
int64_t interval = 20 * 1000; int64_t interval = 20 * 1000;
...@@ -91,11 +92,11 @@ TEST(TD_STREAM_UPDATE_TEST, update) { ...@@ -91,11 +92,11 @@ TEST(TD_STREAM_UPDATE_TEST, update) {
} }
SUpdateInfo *pSU4 = updateInfoInit(-1, TSDB_TIME_PRECISION_MILLI, -1); SUpdateInfo *pSU4 = updateInfoInit(-1, TSDB_TIME_PRECISION_MILLI, -1);
GTEST_ASSERT_EQ(pSU4->watermark, 120 * pSU4->interval); GTEST_ASSERT_EQ(pSU4->watermark, MAX_NUM_SCALABLE_BF * pSU4->interval);
GTEST_ASSERT_EQ(pSU4->interval, MILLISECOND_PER_MINUTE); GTEST_ASSERT_EQ(pSU4->interval, MILLISECOND_PER_MINUTE);
SUpdateInfo *pSU5 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); SUpdateInfo *pSU5 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0);
GTEST_ASSERT_EQ(pSU5->watermark, 120 * pSU4->interval); GTEST_ASSERT_EQ(pSU5->watermark, MAX_NUM_SCALABLE_BF * pSU4->interval);
GTEST_ASSERT_EQ(pSU5->interval, MILLISECOND_PER_MINUTE); GTEST_ASSERT_EQ(pSU5->interval, MILLISECOND_PER_MINUTE);
......
...@@ -411,7 +411,7 @@ SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) { ...@@ -411,7 +411,7 @@ SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) {
} }
uint32_t len; uint32_t len;
char* data = NULL; char* data = NULL;
if (tDecodeBinary(&decoder, (const uint8_t**)(&data), &len) < 0) { if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) {
return NULL; return NULL;
} }
assert(len = pMsg->dataLen); assert(len = pMsg->dataLen);
...@@ -670,7 +670,7 @@ SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) { ...@@ -670,7 +670,7 @@ SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) {
} }
uint32_t len; uint32_t len;
char* data = NULL; char* data = NULL;
if (tDecodeBinary(&decoder, (const uint8_t**)(&data), &len) < 0) { if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) {
return NULL; return NULL;
} }
assert(len = pMsg->dataLen); assert(len = pMsg->dataLen);
......
...@@ -18,14 +18,16 @@ if(USE_TD_MEMORY) ...@@ -18,14 +18,16 @@ if(USE_TD_MEMORY)
add_definitions(-DUSE_TD_MEMORY) add_definitions(-DUSE_TD_MEMORY)
endif () endif ()
if(BUILD_ADDR2LINE) if(BUILD_ADDR2LINE)
if(NOT TD_WINDOWS)
target_include_directories( target_include_directories(
os os
PUBLIC "${TD_SOURCE_DIR}/contrib/libdwarf/src/lib/libdwarf" PUBLIC "${TD_SOURCE_DIR}/contrib/libdwarf/src/lib/libdwarf"
) )
add_definitions(-DUSE_ADDR2LINE)
target_link_libraries( target_link_libraries(
os PUBLIC addr2line dl z os PUBLIC addr2line dl z
) )
endif()
add_definitions(-DUSE_ADDR2LINE)
endif () endif ()
if(CHECK_STR2INT_ERROR) if(CHECK_STR2INT_ERROR)
add_definitions(-DTD_CHECK_STR_TO_INT_ERROR) add_definitions(-DTD_CHECK_STR_TO_INT_ERROR)
......
...@@ -91,7 +91,12 @@ void taosRemoveDir(const char *dirname) { ...@@ -91,7 +91,12 @@ void taosRemoveDir(const char *dirname) {
bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); } bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); }
int32_t taosMkDir(const char *dirname) { int32_t taosMkDir(const char *dirname) {
if (taosDirExist(dirname)) return 0;
#ifdef WINDOWS
int32_t code = _mkdir(dirname, 0755);
#else
int32_t code = mkdir(dirname, 0755); int32_t code = mkdir(dirname, 0755);
#endif
if (code < 0 && errno == EEXIST) { if (code < 0 && errno == EEXIST) {
return 0; return 0;
} }
...@@ -101,36 +106,48 @@ int32_t taosMkDir(const char *dirname) { ...@@ -101,36 +106,48 @@ int32_t taosMkDir(const char *dirname) {
int32_t taosMulMkDir(const char *dirname) { int32_t taosMulMkDir(const char *dirname) {
if (dirname == NULL) return -1; if (dirname == NULL) return -1;
char * temp = strdup(dirname); char temp[1024];
#ifdef WINDOWS
taosRealPath(dirname, temp, sizeof(temp));
#else
strcpy(temp, dirname);
#endif
char * pos = temp; char * pos = temp;
int32_t code = 0; int32_t code = 0;
if (strncmp(temp, "/", 1) == 0) { if (taosDirExist(temp)) return code;
if (strncmp(temp, TD_DIRSEP, 1) == 0) {
pos += 1; pos += 1;
} else if (strncmp(temp, "./", 2) == 0) { } else if (strncmp(temp, "." TD_DIRSEP, 2) == 0) {
pos += 2; pos += 2;
} }
for (; *pos != '\0'; pos++) { for (; *pos != '\0'; pos++) {
if (*pos == '/') { if (*pos == TD_DIRSEP[0]) {
*pos = '\0'; *pos = '\0';
#ifdef WINDOWS
code = _mkdir(temp, 0755);
#else
code = mkdir(temp, 0755); code = mkdir(temp, 0755);
#endif
if (code < 0 && errno != EEXIST) { if (code < 0 && errno != EEXIST) {
free(temp);
return code; return code;
} }
*pos = '/'; *pos = TD_DIRSEP[0];
} }
} }
if (*(pos - 1) != '/') { if (*(pos - 1) != TD_DIRSEP[0]) {
#ifdef WINDOWS
code = _mkdir(temp, 0755);
#else
code = mkdir(temp, 0755); code = mkdir(temp, 0755);
#endif
if (code < 0 && errno != EEXIST) { if (code < 0 && errno != EEXIST) {
free(temp);
return code; return code;
} }
} }
free(temp);
// int32_t code = mkdir(dirname, 0755); // int32_t code = mkdir(dirname, 0755);
if (code < 0 && errno == EEXIST) { if (code < 0 && errno == EEXIST) {
...@@ -233,8 +250,14 @@ char *taosDirName(char *name) { ...@@ -233,8 +250,14 @@ char *taosDirName(char *name) {
_splitpath(name, Drive1, Dir1, NULL, NULL); _splitpath(name, Drive1, Dir1, NULL, NULL);
size_t dirNameLen = strlen(Drive1) + strlen(Dir1); size_t dirNameLen = strlen(Drive1) + strlen(Dir1);
if (dirNameLen > 0) { if (dirNameLen > 0) {
if (name[dirNameLen - 1] == '/' || name[dirNameLen - 1] == '\\') {
name[dirNameLen - 1] = 0;
} else {
name[dirNameLen] = 0; name[dirNameLen] = 0;
} }
} else {
name[0] = 0;
}
return name; return name;
#else #else
return dirname(name); return dirname(name);
......
...@@ -109,8 +109,11 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha ...@@ -109,8 +109,11 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha
int64_t taosCopyFile(const char *from, const char *to) { int64_t taosCopyFile(const char *from, const char *to) {
#ifdef WINDOWS #ifdef WINDOWS
assert(0); if (CopyFile(from, to, 0)) {
return 1;
} else {
return -1; return -1;
}
#else #else
char buffer[4096]; char buffer[4096];
int64_t size = 0; int64_t size = 0;
...@@ -343,7 +346,11 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { ...@@ -343,7 +346,11 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
char *tbuf = (char *)buf; char *tbuf = (char *)buf;
while (leftbytes > 0) { while (leftbytes > 0) {
#ifdef WINDOWS
readbytes = _read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes);
#else
readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes);
#endif
if (readbytes < 0) { if (readbytes < 0) {
if (errno == EINTR) { if (errno == EINTR) {
continue; continue;
...@@ -379,10 +386,10 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) ...@@ -379,10 +386,10 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
#endif #endif
assert(pFile->fd >= 0); // Please check if you have closed the file. assert(pFile->fd >= 0); // Please check if you have closed the file.
#ifdef WINDOWS #ifdef WINDOWS
size_t pos = lseek(pFile->fd, 0, SEEK_CUR); size_t pos = _lseek(pFile->fd, 0, SEEK_CUR);
lseek(pFile->fd, offset, SEEK_SET); _lseek(pFile->fd, offset, SEEK_SET);
int64_t ret = read(pFile->fd, buf, count); int64_t ret = _read(pFile->fd, buf, count);
lseek(pFile->fd, pos, SEEK_SET); _lseek(pFile->fd, pos, SEEK_SET);
#else #else
int64_t ret = pread(pFile->fd, buf, count, offset); int64_t ret = pread(pFile->fd, buf, count, offset);
#endif #endif
...@@ -428,7 +435,11 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { ...@@ -428,7 +435,11 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
taosThreadRwlockRdlock(&(pFile->rwlock)); taosThreadRwlockRdlock(&(pFile->rwlock));
#endif #endif
assert(pFile->fd >= 0); // Please check if you have closed the file. assert(pFile->fd >= 0); // Please check if you have closed the file.
#ifdef WINDOWS
int64_t ret = _lseek(pFile->fd, offset, whence);
#else
int64_t ret = lseek(pFile->fd, offset, whence); int64_t ret = lseek(pFile->fd, offset, whence);
#endif
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock)); taosThreadRwlockUnlock(&(pFile->rwlock));
#endif #endif
...@@ -567,12 +578,12 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in ...@@ -567,12 +578,12 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
#ifdef WINDOWS #ifdef WINDOWS
lseek(pFileIn->fd, (int32_t)(*offset), 0); _lseek(pFileIn->fd, (int32_t)(*offset), 0);
int64_t writeLen = 0; int64_t writeLen = 0;
uint8_t buffer[_SEND_FILE_STEP_] = {0}; uint8_t buffer[_SEND_FILE_STEP_] = {0};
for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); size_t rlen = _read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_);
if (rlen <= 0) { if (rlen <= 0) {
return writeLen; return writeLen;
} else if (rlen < _SEND_FILE_STEP_) { } else if (rlen < _SEND_FILE_STEP_) {
...@@ -586,7 +597,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in ...@@ -586,7 +597,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
int64_t remain = size - writeLen; int64_t remain = size - writeLen;
if (remain > 0) { if (remain > 0) {
size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain); size_t rlen = _read(pFileIn->fd, (void *)buffer, (size_t)remain);
if (rlen <= 0) { if (rlen <= 0) {
return writeLen; return writeLen;
} else { } else {
......
...@@ -37,6 +37,49 @@ typedef struct TdMemoryInfo { ...@@ -37,6 +37,49 @@ typedef struct TdMemoryInfo {
#ifdef WINDOWS #ifdef WINDOWS
#define tstrdup(str) _strdup(str) #define tstrdup(str) _strdup(str)
int32_t taosBackTrace(void **buffer, int32_t size) {
int32_t frame = 0;
return frame;
}
#ifdef USE_ADDR2LINE
#include <DbgHelp.h>
#pragma comment(lib, "dbghelp.lib")
void taosPrintBackTrace() {
#define MAX_STACK_FRAMES 20
void *pStack[MAX_STACK_FRAMES];
HANDLE process = GetCurrentProcess();
SymInitialize(process, NULL, TRUE);
WORD frames = CaptureStackBackTrace(1, MAX_STACK_FRAMES, pStack, NULL);
char buf_tmp[1024];
for (WORD i = 0; i < frames; ++i) {
DWORD64 address = (DWORD64)(pStack[i]);
DWORD64 displacementSym = 0;
char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;
pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
pSymbol->MaxNameLen = MAX_SYM_NAME;
DWORD displacementLine = 0;
IMAGEHLP_LINE64 line;
//SymSetOptions(SYMOPT_LOAD_LINES);
line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
if (SymFromAddr(process, address, &displacementSym, pSymbol) && SymGetLineFromAddr64(process, address, &displacementLine, &line)) {
snprintf(buf_tmp,sizeof(buf_tmp),"BackTrace %08" PRId64 " %s:%d %s\n", taosGetSelfPthreadId(), line.FileName, line.LineNumber, pSymbol->Name);
} else {
snprintf(buf_tmp,sizeof(buf_tmp),"BackTrace error: %d\n",GetLastError());
}
write(1,buf_tmp,strlen(buf_tmp));
}
}
#endif
#else #else
#define tstrdup(str) strdup(str) #define tstrdup(str) strdup(str)
......
...@@ -68,9 +68,32 @@ int32_t tsem_wait(tsem_t* sem) { ...@@ -68,9 +68,32 @@ int32_t tsem_wait(tsem_t* sem) {
} }
int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) {
int ret = 0; struct timespec ts, rel;
FILETIME ft_before, ft_after;
return ret; int rc;
rel.tv_sec = 0;
rel.tv_nsec = nanosecs;
GetSystemTimeAsFileTime(&ft_before);
errno = 0;
rc = sem_timedwait(&sem, pthread_win32_getabstime_np(&ts, &rel));
/* This should have timed out */
assert(errno == ETIMEDOUT);
assert(rc != 0);
GetSystemTimeAsFileTime(&ft_after);
// We specified a non-zero wait. Time must advance.
if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime)
{
printf("nanoseconds: %d, rc: %d, errno: %d. before filetime: %d, %d; after filetime: %d, %d\n",
nanosecs, rc, errno,
(int)ft_before.dwLowDateTime, (int)ft_before.dwHighDateTime,
(int)ft_after.dwLowDateTime, (int)ft_after.dwHighDateTime);
printf("time must advance during sem_timedwait.");
return 1;
}
return 0;
} }
#elif defined(_TD_DARWIN_64) #elif defined(_TD_DARWIN_64)
......
...@@ -718,7 +718,11 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { ...@@ -718,7 +718,11 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) {
bzero((char *)&serverAdd, sizeof(serverAdd)); bzero((char *)&serverAdd, sizeof(serverAdd));
serverAdd.sin_family = AF_INET; serverAdd.sin_family = AF_INET;
#ifdef WINDOWS
serverAdd.sin_addr.s_addr = INADDR_ANY;
#else
serverAdd.sin_addr.s_addr = ip; serverAdd.sin_addr.s_addr = ip;
#endif
serverAdd.sin_port = (uint16_t)htons(port); serverAdd.sin_port = (uint16_t)htons(port);
if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) {
...@@ -882,6 +886,16 @@ void taosBlockSIGPIPE() { ...@@ -882,6 +886,16 @@ void taosBlockSIGPIPE() {
} }
uint32_t taosGetIpv4FromFqdn(const char *fqdn) { uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
#ifdef WINDOWS
// Initialize Winsock
WSADATA wsaData;
int iResult;
iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (iResult != 0) {
printf("WSAStartup failed: %d\n", iResult);
return 1;
}
#endif
struct addrinfo hints = {0}; struct addrinfo hints = {0};
hints.ai_family = AF_INET; hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM; hints.ai_socktype = SOCK_STREAM;
...@@ -899,12 +913,12 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { ...@@ -899,12 +913,12 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
} else { } else {
#ifdef EAI_SYSTEM #ifdef EAI_SYSTEM
if (ret == EAI_SYSTEM) { if (ret == EAI_SYSTEM) {
// printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); printf("failed to get the ip address, fqdn:%s, errno:%d, since:%s", fqdn, errno, strerror(errno));
} else { } else {
// printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret));
} }
#else #else
// printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret));
#endif #endif
return 0xFFFFFFFF; return 0xFFFFFFFF;
} }
......
...@@ -30,7 +30,7 @@ struct SEncoderNode { ...@@ -30,7 +30,7 @@ struct SEncoderNode {
struct SDecoderNode { struct SDecoderNode {
SDecoderNode* pNext; SDecoderNode* pNext;
const uint8_t* data; uint8_t* data;
uint32_t size; uint32_t size;
uint32_t pos; uint32_t pos;
}; };
...@@ -52,7 +52,7 @@ void tEncoderClear(SEncoder* pCoder) { ...@@ -52,7 +52,7 @@ void tEncoderClear(SEncoder* pCoder) {
memset(pCoder, 0, sizeof(*pCoder)); memset(pCoder, 0, sizeof(*pCoder));
} }
void tDecoderInit(SDecoder* pDecoder, const uint8_t* data, uint32_t size) { void tDecoderInit(SDecoder* pDecoder, uint8_t* data, uint32_t size) {
pDecoder->data = data; pDecoder->data = data;
pDecoder->size = size; pDecoder->size = size;
pDecoder->pos = 0; pDecoder->pos = 0;
......
...@@ -244,6 +244,7 @@ class TDTestCase: ...@@ -244,6 +244,7 @@ class TDTestCase:
if user is None: if user is None:
user = self.root_user user = self.root_user
with taos_connect(user=user.name, passwd=user.passwd) as use: with taos_connect(user=user.name, passwd=user.passwd) as use:
time.sleep(2)
use.query("use db") use.query("use db")
use.query("show tables") use.query("show tables")
if check_priv == PRIVILEGES_ALL: if check_priv == PRIVILEGES_ALL:
...@@ -398,6 +399,7 @@ class TDTestCase: ...@@ -398,6 +399,7 @@ class TDTestCase:
tdLog.printNoPrefix("==========step 1.18: revoke all from all = nothing") tdLog.printNoPrefix("==========step 1.18: revoke all from all = nothing")
self.revoke_user(user=self.users[2], priv=PRIVILEGES_ALL) self.revoke_user(user=self.users[2], priv=PRIVILEGES_ALL)
time.sleep(3)
self.__user_check(user=self.users[2], check_priv=None) self.__user_check(user=self.users[2], check_priv=None)
def __grant_err(self): def __grant_err(self):
......
Subproject commit 0aad27d725f4ee6b18daf1db0c07d933aed16eea Subproject commit 788929bdc475d264d8306ceff30f7df006fd18d8
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册