Skip to content

TDengine
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
TDengine
TDengine
提交 39eec5e8 编写于 作者: G gccgdb1234
浏览文件
操作

Merge branch 'main' of github.com:taosdata/TDengine into TD-13572

上级 6b3fa3f5 8edceef2
展开全部 显示空白变更内容
内联 并排
Showing with 1968 addition and 1051 deletion
cmake/taostools_CMakeLists.txt.in
浏览文件 @ 39eec5e8
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 41affde GIT_TAG ad1a32b
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE
......
docs/zh/05-get-started/index.md
浏览文件 @ 39eec5e8
...@@ -41,7 +41,7 @@ TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概 ...@@ -41,7 +41,7 @@ TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概
<td style={{padding:'1em 3em',border:0}}><img src={official_account} alt="TDengine 微信公众号" width="200" /></td> <td style={{padding:'1em 3em',border:0}}><img src={official_account} alt="TDengine 微信公众号" width="200" /></td>
</tr> </tr>
<tr align="center"> <tr align="center">
<td style={{padding:'1em 3em',border:0}}>加入“物联网大数据技术群”<br/>与大家进行技术交流</td> <td style={{padding:'1em 3em',border:0}}>加入 TDengine 微信群<br/>了解学习最新物联网技术</td>
<td style={{padding:'1em 3em',border:0}}>关注 TDengine 视频号<br/>收看技术直播与教学视频</td> <td style={{padding:'1em 3em',border:0}}>关注 TDengine 视频号<br/>收看技术直播与教学视频</td>
<td style={{padding:'1em 3em',border:0}}>关注 TDengine 公众号<br/>阅读技术文章与行业案例</td> <td style={{padding:'1em 3em',border:0}}>关注 TDengine 公众号<br/>阅读技术文章与行业案例</td>
</tr> </tr>
......
include/common/tcommon.h
浏览文件 @ 39eec5e8
...@@ -25,13 +25,6 @@ ...@@ -25,13 +25,6 @@
extern "C" { extern "C" {
#endif #endif
// TODO remove it
enum {
TMQ_CONF__RESET_OFFSET__NONE = -3,
TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
TMQ_CONF__RESET_OFFSET__LATEST = -1,
};
// clang-format off // clang-format off
#define IS_META_MSG(x) ( \ #define IS_META_MSG(x) ( \
x == TDMT_VND_CREATE_STB \ x == TDMT_VND_CREATE_STB \
......
include/common/tglobal.h
浏览文件 @ 39eec5e8
...@@ -99,6 +99,7 @@ extern char *tsSvrCrashReportUri; ...@@ -99,6 +99,7 @@ extern char *tsSvrCrashReportUri;
// query buffer management // query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node
extern int32_t tsCacheLazyLoadThreshold; // cost threshold for last/last_row loading cache as much as possible
// query client // query client
extern int32_t tsQueryPolicy; extern int32_t tsQueryPolicy;
...@@ -147,8 +148,8 @@ extern char tsUdfdLdLibPath[]; ...@@ -147,8 +148,8 @@ extern char tsUdfdLdLibPath[];
// schemaless // schemaless
extern char tsSmlChildTableName[]; extern char tsSmlChildTableName[];
extern char tsSmlTagName[]; extern char tsSmlTagName[];
//extern bool tsSmlDataFormat; // extern bool tsSmlDataFormat;
//extern int32_t tsSmlBatchSize; // extern int32_t tsSmlBatchSize;
// wal // wal
extern int64_t tsWalFsyncDataSizeLimit; extern int64_t tsWalFsyncDataSizeLimit;
......
include/common/tmsg.h
浏览文件 @ 39eec5e8
...@@ -1136,6 +1136,7 @@ typedef struct { ...@@ -1136,6 +1136,7 @@ typedef struct {
int64_t numOfInsertSuccessReqs; int64_t numOfInsertSuccessReqs;
int64_t numOfBatchInsertReqs; int64_t numOfBatchInsertReqs;
int64_t numOfBatchInsertSuccessReqs; int64_t numOfBatchInsertSuccessReqs;
int32_t numOfCachedTables;
} SVnodeLoad; } SVnodeLoad;
typedef struct { typedef struct {
...@@ -3185,6 +3186,7 @@ typedef struct { ...@@ -3185,6 +3186,7 @@ typedef struct {
SArray* blockData; SArray* blockData;
SArray* blockTbName; SArray* blockTbName;
SArray* blockSchema; SArray* blockSchema;
// the following attributes are extended from SMqDataRsp
int32_t createTableNum; int32_t createTableNum;
SArray* createTableLen; SArray* createTableLen;
SArray* createTableReq; SArray* createTableReq;
......
include/libs/sync/sync.h
浏览文件 @ 39eec5e8
...@@ -143,10 +143,11 @@ typedef struct SSyncFSM { ...@@ -143,10 +143,11 @@ typedef struct SSyncFSM {
void* data; void* data;
int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
SyncIndex (*FpAppliedIndexCb)(const struct SSyncFSM* pFsm);
int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm); void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm, const SyncIndex commitIdx);
void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SReConfigCbMeta* pMeta); void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SReConfigCbMeta* pMeta);
void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm); bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm);
......
include/util/taoserror.h
浏览文件 @ 39eec5e8
...@@ -542,7 +542,8 @@ int32_t* taosGetErrno(); ...@@ -542,7 +542,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) #define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913)
#define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914) #define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914)
#define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal #define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal
#define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) // #define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916)
#define TSDB_CODE_SYN_WRITE_STALL TAOS_DEF_ERROR_CODE(0, 0x0917)
#define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF)
// tq // tq
......
include/util/tlrucache.h
浏览文件 @ 39eec5e8
...@@ -55,6 +55,8 @@ void *taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle); ...@@ -55,6 +55,8 @@ void *taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle);
size_t taosLRUCacheGetUsage(SLRUCache *cache); size_t taosLRUCacheGetUsage(SLRUCache *cache);
size_t taosLRUCacheGetPinnedUsage(SLRUCache *cache); size_t taosLRUCacheGetPinnedUsage(SLRUCache *cache);
int32_t taosLRUCacheGetElems(SLRUCache *cache);
void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity); void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity);
size_t taosLRUCacheGetCapacity(SLRUCache *cache); size_t taosLRUCacheGetCapacity(SLRUCache *cache);
......
packaging/cfg/taos.cfg
浏览文件 @ 39eec5e8
######################################################## ########################################################
# # # #
# TDengine Configuration # # Configuration #
# Any questions, please email support@taosdata.com # # Any questions, please email support@taosdata.com #
# # # #
######################################################## ########################################################
######### 0. Client only configurations ############# ######### 0. Client only configurations #############
# The interval for TDengine CLI to send heartbeat to mnode # The interval for CLI to send heartbeat to mnode
# shellActivityTimer 3 # shellActivityTimer 3
############### 1. Cluster End point ############################ ############### 1. Cluster End point ############################
# The end point of the first dnode in the cluster to be connected to when this dnode or a TDengine CLI `taos` is started # The end point of the first dnode in the cluster to be connected to when this dnode or a CLI `taos` is started
# firstEp hostname:6030 # firstEp hostname:6030
# The end point of the second dnode to be connected to if the firstEp is not available # The end point of the second dnode to be connected to if the firstEp is not available
...@@ -40,10 +40,10 @@ ...@@ -40,10 +40,10 @@
# temporary file's directory, if you are using Windows platform please change to Windows path # temporary file's directory, if you are using Windows platform please change to Windows path
# tempDir /tmp/ # tempDir /tmp/
# Switch for allowing TDengine to collect and report service usage information # Switch for allowing to collect and report service usage information
# telemetryReporting 1 # telemetryReporting 1
# Switch for allowing TDengine to collect and report crash information # Switch for allowing to collect and report crash information
# crashReporting 1 # crashReporting 1
# The maximum number of vnodes supported by this dnode # The maximum number of vnodes supported by this dnode
......
packaging/debRpmAutoInstall.sh
浏览文件 @ 39eec5e8
#!/usr/bin/expect #!/usr/bin/expect
set packgeName [lindex $argv 0] set packageName [lindex $argv 0]
set packageSuffix [lindex $argv 1] set packageSuffix [lindex $argv 1]
set timeout 3 set timeout 3
if { ${packageSuffix} == "deb" } { if { ${packageSuffix} == "deb" } {
spawn dpkg -i ${packgeName} spawn dpkg -i ${packageName}
} elseif { ${packageSuffix} == "rpm"} { } elseif { ${packageSuffix} == "rpm"} {
spawn rpm -ivh ${packgeName} spawn rpm -ivh ${packageName}
} }
expect "*one:" expect "*one:"
send "\r" send "\r"
......
packaging/testpackage.sh
浏览文件 @ 39eec5e8
...@@ -25,7 +25,7 @@ sourcePath="nas" ...@@ -25,7 +25,7 @@ sourcePath="nas"
cpuType="x64" cpuType="x64"
lite="true" lite="true"
packageType="tar" packageType="tar"
subFile="taos.tar.gz" subFile="package.tar.gz"
while getopts "m:c:f:l:s:o:t:v:h" opt; do while getopts "m:c:f:l:s:o:t:v:h" opt; do
case $opt in case $opt in
m) m)
...@@ -79,9 +79,9 @@ GREEN_UNDERLINE='\033[4;32m' ...@@ -79,9 +79,9 @@ GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m' NC='\033[0m'
if [[ ${verMode} = "enterprise" ]];then if [[ ${verMode} = "enterprise" ]];then
prePackag="TDengine-enterprise-${testFile}" prePackage="TDengine-enterprise-${testFile}"
elif [ ${verMode} = "community" ];then elif [ ${verMode} = "community" ];then
prePackag="TDengine-${testFile}" prePackage="TDengine-${testFile}"
fi fi
if [ ${lite} = "true" ];then if [ ${lite} = "true" ];then
packageLite="-Lite" packageLite="-Lite"
...@@ -92,10 +92,10 @@ if [[ "$packageType" = "tar" ]] ;then ...@@ -92,10 +92,10 @@ if [[ "$packageType" = "tar" ]] ;then
packageType="tar.gz" packageType="tar.gz"
fi fi
tdPath="${prePackag}-${version}" tdPath="${prePackage}-${version}"
originTdpPath="${prePackag}-${originversion}" originTdpPath="${prePackage}-${originversion}"
packgeName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" packageName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}"
originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}" originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}"
if [ "$testFile" == "server" ] ;then if [ "$testFile" == "server" ] ;then
...@@ -105,13 +105,13 @@ elif [ ${testFile} = "client" ];then ...@@ -105,13 +105,13 @@ elif [ ${testFile} = "client" ];then
elif [ ${testFile} = "tools" ];then elif [ ${testFile} = "tools" ];then
tdPath="taosTools-${version}" tdPath="taosTools-${version}"
originTdpPath="taosTools-${originversion}" originTdpPath="taosTools-${originversion}"
packgeName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" packageName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}"
originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}" originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}"
installCmd="install-taostools.sh" installCmd="install-tools.sh"
fi fi
echo "tdPath:${tdPath},originTdpPath:${originTdpPath},packgeName:${packgeName},originPackageName:${originPackageName}" echo "tdPath:${tdPath},originTdpPath:${originTdpPath},packageName:${packageName},originPackageName:${originPackageName}"
function cmdInstall { function cmdInstall {
command=$1 command=$1
if command -v ${command} ;then if command -v ${command} ;then
...@@ -206,7 +206,7 @@ else ...@@ -206,7 +206,7 @@ else
fi fi
if [[ ${packgeName} =~ "server" ]] ;then if [[ ${packageName} =~ "server" ]] ;then
echoColor BD " pkill -9 taosd " echoColor BD " pkill -9 taosd "
pkill -9 taosd pkill -9 taosd
fi fi
...@@ -232,25 +232,25 @@ if [ -d ${installPath}/${tdPath} ] ;then ...@@ -232,25 +232,25 @@ if [ -d ${installPath}/${tdPath} ] ;then
fi fi
echoColor G "===== download installPackage =====" echoColor G "===== download installPackage ====="
cd ${installPath} && wgetFile ${packgeName} ${version} ${sourcePath} cd ${installPath} && wgetFile ${packageName} ${version} ${sourcePath}
cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath} cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath}
cd ${installPath} cd ${installPath}
cp -r ${scriptDir}/debRpmAutoInstall.sh . cp -r ${scriptDir}/debRpmAutoInstall.sh .
packageSuffix=$(echo ${packgeName} | awk -F '.' '{print $NF}') packageSuffix=$(echo ${packageName} | awk -F '.' '{print $NF}')
if [ ! -f debRpmAutoInstall.sh ];then if [ ! -f debRpmAutoInstall.sh ];then
echo '#!/usr/bin/expect ' > debRpmAutoInstall.sh echo '#!/usr/bin/expect ' > debRpmAutoInstall.sh
echo 'set packgeName [lindex $argv 0]' >> debRpmAutoInstall.sh echo 'set packageName [lindex $argv 0]' >> debRpmAutoInstall.sh
echo 'set packageSuffix [lindex $argv 1]' >> debRpmAutoInstall.sh echo 'set packageSuffix [lindex $argv 1]' >> debRpmAutoInstall.sh
echo 'set timeout 3 ' >> debRpmAutoInstall.sh echo 'set timeout 3 ' >> debRpmAutoInstall.sh
echo 'if { ${packageSuffix} == "deb" } {' >> debRpmAutoInstall.sh echo 'if { ${packageSuffix} == "deb" } {' >> debRpmAutoInstall.sh
echo ' spawn dpkg -i ${packgeName} ' >> debRpmAutoInstall.sh echo ' spawn dpkg -i ${packageName} ' >> debRpmAutoInstall.sh
echo '} elseif { ${packageSuffix} == "rpm"} {' >> debRpmAutoInstall.sh echo '} elseif { ${packageSuffix} == "rpm"} {' >> debRpmAutoInstall.sh
echo ' spawn rpm -ivh ${packgeName}' >> debRpmAutoInstall.sh echo ' spawn rpm -ivh ${packageName}' >> debRpmAutoInstall.sh
echo '}' >> debRpmAutoInstall.sh echo '}' >> debRpmAutoInstall.sh
echo 'expect "*one:"' >> debRpmAutoInstall.sh echo 'expect "*one:"' >> debRpmAutoInstall.sh
echo 'send "\r"' >> debRpmAutoInstall.sh echo 'send "\r"' >> debRpmAutoInstall.sh
...@@ -261,25 +261,25 @@ fi ...@@ -261,25 +261,25 @@ fi
echoColor G "===== instal Package =====" echoColor G "===== instal Package ====="
if [[ ${packgeName} =~ "deb" ]];then if [[ ${packageName} =~ "deb" ]];then
cd ${installPath} cd ${installPath}
dpkg -r taostools dpkg -r taostools
dpkg -r tdengine dpkg -r tdengine
if [[ ${packgeName} =~ "TDengine" ]];then if [[ ${packageName} =~ "TDengine" ]];then
echoColor BD "./debRpmAutoInstall.sh ${packgeName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packgeName} ${packageSuffix} echoColor BD "./debRpmAutoInstall.sh ${packageName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packageName} ${packageSuffix}
else else
echoColor BD "dpkg -i ${packgeName}" && dpkg -i ${packgeName} echoColor BD "dpkg -i ${packageName}" && dpkg -i ${packageName}
fi fi
elif [[ ${packgeName} =~ "rpm" ]];then elif [[ ${packageName} =~ "rpm" ]];then
cd ${installPath} cd ${installPath}
sudo rpm -e tdengine sudo rpm -e tdengine
sudo rpm -e taostools sudo rpm -e taostools
if [[ ${packgeName} =~ "TDengine" ]];then if [[ ${packageName} =~ "TDengine" ]];then
echoColor BD "./debRpmAutoInstall.sh ${packgeName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packgeName} ${packageSuffix} echoColor BD "./debRpmAutoInstall.sh ${packageName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packageName} ${packageSuffix}
else else
echoColor BD "rpm -ivh ${packgeName}" && rpm -ivh ${packgeName} echoColor BD "rpm -ivh ${packageName}" && rpm -ivh ${packageName}
fi fi
elif [[ ${packgeName} =~ "tar" ]];then elif [[ ${packageName} =~ "tar" ]];then
echoColor G "===== check installPackage File of tar =====" echoColor G "===== check installPackage File of tar ====="
cd ${oriInstallPath} cd ${oriInstallPath}
if [ ! -f {originPackageName} ];then if [ ! -f {originPackageName} ];then
...@@ -290,7 +290,7 @@ elif [[ ${packgeName} =~ "tar" ]];then ...@@ -290,7 +290,7 @@ elif [[ ${packgeName} =~ "tar" ]];then
echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName}
cd ${installPath} cd ${installPath}
echoColor YD "unzip the new installation package" echoColor YD "unzip the new installation package"
echoColor BD "tar -xf ${packgeName}" && tar -xf ${packgeName} echoColor BD "tar -xf ${packageName}" && tar -xf ${packageName}
if [ ${testFile} != "tools" ] ;then if [ ${testFile} != "tools" ] ;then
cd ${installPath}/${tdPath} && tar xf ${subFile} cd ${installPath}/${tdPath} && tar xf ${subFile}
...@@ -326,15 +326,15 @@ fi ...@@ -326,15 +326,15 @@ fi
cd ${installPath} cd ${installPath}
if [[ ${packgeName} =~ "Lite" ]] || ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) || ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]]) || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then if [[ ${packageName} =~ "Lite" ]] || ([[ ${packageName} =~ "x64" ]] && [[ ${packageName} =~ "client" ]]) || ([[ ${packageName} =~ "deb" ]] && [[ ${packageName} =~ "server" ]]) || ([[ ${packageName} =~ "rpm" ]] && [[ ${packageName} =~ "server" ]]) ;then
echoColor G "===== install taos-tools when package is lite or client =====" echoColor G "===== install taos-tools when package is lite or client ====="
cd ${installPath} cd ${installPath}
if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then
wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web
tar xf taosTools-2.1.3-Linux-x64.tar.gz tar xf taosTools-2.1.3-Linux-x64.tar.gz
fi fi
cd taosTools-2.1.3 && bash install-taostools.sh cd taosTools-2.1.3 && bash install-tools.sh
elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then elif ([[ ${packageName} =~ "arm64" ]] && [[ ${packageName} =~ "client" ]]);then
echoColor G "===== install taos-tools arm when package is arm64-client =====" echoColor G "===== install taos-tools arm when package is arm64-client ====="
cd ${installPath} cd ${installPath}
if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then
...@@ -342,37 +342,37 @@ elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then ...@@ -342,37 +342,37 @@ elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then
tar xf taosTools-2.1.3-Linux-arm64.tar.gz tar xf taosTools-2.1.3-Linux-arm64.tar.gz
fi fi
cd taosTools-2.1.3 && bash install-taostools.sh cd taosTools-2.1.3 && bash install-tools.sh
fi fi
echoColor G "===== start TDengine =====" echoColor G "===== start TDengine ====="
if [[ ${packgeName} =~ "server" ]] ;then if [[ ${packageName} =~ "server" ]] ;then
echoColor BD " rm -rf /var/lib/taos/* && systemctl restart taosd " echoColor BD " rm -rf /var/lib/taos/* && systemctl restart taosd "
rm -rf /var/lib/taos/* rm -rf /var/lib/taos/*
systemctl restart taosd systemctl restart taosd
fi fi
rm -rf ${installPath}/${packgeName} rm -rf ${installPath}/${packageName}
rm -rf ${installPath}/${tdPath}/ rm -rf ${installPath}/${tdPath}/
# if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then # if ([[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "tar" ]]) || [[ ${packageName} =~ "client" ]] ;then
# echoColor G "===== install taos-tools when package is lite or client =====" # echoColor G "===== install taos-tools when package is lite or client ====="
# cd ${installPath} # cd ${installPath}
# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . # wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
# tar xf taosTools-2.1.2-Linux-x64.tar.gz # tar xf taosTools-2.1.2-Linux-x64.tar.gz
# cd taosTools-2.1.2 && bash install-taostools.sh # cd taosTools-2.1.2 && bash install-tools.sh
# elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "deb" ]] ;then # elif [[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "deb" ]] ;then
# echoColor G "===== install taos-tools when package is lite or client =====" # echoColor G "===== install taos-tools when package is lite or client ====="
# cd ${installPath} # cd ${installPath}
# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . # wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
# tar xf taosTools-2.1.2-Linux-x64.tar.gz # tar xf taosTools-2.1.2-Linux-x64.tar.gz
# cd taosTools-2.1.2 && bash install-taostools.sh # cd taosTools-2.1.2 && bash install-tools.sh
# elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "rpm" ]] ;then # elif [[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "rpm" ]] ;then
# echoColor G "===== install taos-tools when package is lite or client =====" # echoColor G "===== install taos-tools when package is lite or client ====="
# cd ${installPath} # cd ${installPath}
# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . # wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
# tar xf taosTools-2.1.2-Linux-x64.tar.gz # tar xf taosTools-2.1.2-Linux-x64.tar.gz
# cd taosTools-2.1.2 && bash install-taostools.sh # cd taosTools-2.1.2 && bash install-tools.sh
# fi # fi
packaging/tools/install.sh
浏览文件 @ 39eec5e8
...@@ -24,7 +24,7 @@ productName="TDengine" ...@@ -24,7 +24,7 @@ productName="TDengine"
emailName="taosdata.com" emailName="taosdata.com"
uninstallScript="rmtaos" uninstallScript="rmtaos"
historyFile="taos_history" historyFile="taos_history"
tarName="taos.tar.gz" tarName="package.tar.gz"
dataDir="/var/lib/taos" dataDir="/var/lib/taos"
logDir="/var/log/taos" logDir="/var/log/taos"
configDir="/etc/taos" configDir="/etc/taos"
...@@ -222,24 +222,24 @@ function install_bin() { ...@@ -222,24 +222,24 @@ function install_bin() {
${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/*
#Make link #Make link
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
[ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
[ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -s ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || : [ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || :
[ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || :
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
[ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
[ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || : [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || :
[ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -s ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || : [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || :
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || :
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || :
[ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || :
fi fi
} }
...@@ -250,14 +250,14 @@ function install_lib() { ...@@ -250,14 +250,14 @@ function install_lib() {
#${csudo}rm -rf ${v15_java_app_dir} || : #${csudo}rm -rf ${v15_java_app_dir} || :
${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
[ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
[ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : [ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
fi fi
...@@ -347,10 +347,10 @@ function install_header() { ...@@ -347,10 +347,10 @@ function install_header() {
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || :
${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*
${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo}ln -sf ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -sf ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -sf ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h ${csudo}ln -sf ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
[ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || : [ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || :
} }
...@@ -511,7 +511,7 @@ function install_adapter_config() { ...@@ -511,7 +511,7 @@ function install_adapter_config() {
fi fi
[ -f ${cfg_install_dir}/${adapterName}.toml ] && [ -f ${cfg_install_dir}/${adapterName}.toml ] &&
${csudo}ln -s ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml ${csudo}ln -sf ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml
[ ! -z $1 ] && return 0 || : # only install client [ ! -z $1 ] && return 0 || : # only install client
...@@ -527,7 +527,7 @@ function install_config() { ...@@ -527,7 +527,7 @@ function install_config() {
${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new
fi fi
${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg ${csudo}ln -sf ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg
[ ! -z $1 ] && return 0 || : # only install client [ ! -z $1 ] && return 0 || : # only install client
...@@ -573,13 +573,13 @@ function install_log() { ...@@ -573,13 +573,13 @@ function install_log() {
${csudo}rm -rf ${log_dir} || : ${csudo}rm -rf ${log_dir} || :
${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
${csudo}ln -s ${log_dir} ${install_main_dir}/log ${csudo}ln -sf ${log_dir} ${install_main_dir}/log
} }
function install_data() { function install_data() {
${csudo}mkdir -p ${data_dir} ${csudo}mkdir -p ${data_dir}
${csudo}ln -s ${data_dir} ${install_main_dir}/data ${csudo}ln -sf ${data_dir} ${install_main_dir}/data
} }
function install_connector() { function install_connector() {
...@@ -862,21 +862,21 @@ function updateProduct() { ...@@ -862,21 +862,21 @@ function updateProduct() {
openresty_work=false openresty_work=false
echo echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml" echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml"
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}" echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}"
else else
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \ [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start ${clientName} Adapter ${NC}: taosadapter &${NC}" echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}"
fi fi
if [ ${openresty_work} = 'true' ]; then if [ ${openresty_work} = 'true' ]; then
...@@ -887,7 +887,7 @@ function updateProduct() { ...@@ -887,7 +887,7 @@ function updateProduct() {
if ((${prompt_force} == 1)); then if ((${prompt_force} == 1)); then
echo "" echo ""
echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName} $exist_version!${NC}" echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}"
fi fi
echo echo
echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}"
...@@ -944,21 +944,21 @@ function installProduct() { ...@@ -944,21 +944,21 @@ function installProduct() {
# Ask if to start the service # Ask if to start the service
echo echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml"
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}"
else else
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \ [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}"
fi fi
if [ ! -z "$firstEp" ]; then if [ ! -z "$firstEp" ]; then
...@@ -970,24 +970,24 @@ function installProduct() { ...@@ -970,24 +970,24 @@ function installProduct() {
tmpPort="" tmpPort=""
fi fi
if [[ "$tmpPort" != "" ]]; then if [[ "$tmpPort" != "" ]]; then
echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else else
echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo echo
elif [ ! -z "$serverFqdn" ]; then elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $serverFqdn${GREEN_DARK} to login into ${productName} server${NC}" echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}"
echo echo
fi fi
echo -e "\033[44;32;1m${productName} is installed successfully!${NC}" echo -e "\033[44;32;1m${productName2} is installed successfully!${NC}"
echo echo
else # Only install client else # Only install client
install_bin install_bin
install_config install_config
echo echo
echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}" echo -e "\033[44;32;1m${productName2} client is installed successfully!${NC}"
fi fi
touch ~/.${historyFile} touch ~/.${historyFile}
......
packaging/tools/install_client.sh
浏览文件 @ 39eec5e8
...@@ -17,7 +17,7 @@ serverName="taosd" ...@@ -17,7 +17,7 @@ serverName="taosd"
clientName="taos" clientName="taos"
uninstallScript="rmtaos" uninstallScript="rmtaos"
configFile="taos.cfg" configFile="taos.cfg"
tarName="taos.tar.gz" tarName="package.tar.gz"
osType=Linux osType=Linux
pagMode=full pagMode=full
......
packaging/tools/make_install.sh
浏览文件 @ 39eec5e8
...@@ -606,23 +606,23 @@ function update_TDengine() { ...@@ -606,23 +606,23 @@ function update_TDengine() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
else else
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \ [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}"
else else
echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}"
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}"
fi fi
fi fi
...@@ -658,23 +658,23 @@ function install_TDengine() { ...@@ -658,23 +658,23 @@ function install_TDengine() {
echo echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
else else
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \ [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}"
else else
echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}"
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}"
fi fi
fi fi
......
packaging/tools/makeclient.sh
浏览文件 @ 39eec5e8
...@@ -24,7 +24,7 @@ clientName2="${12}" ...@@ -24,7 +24,7 @@ clientName2="${12}"
productName="TDengine" productName="TDengine"
clientName="taos" clientName="taos"
configFile="taos.cfg" configFile="taos.cfg"
tarName="taos.tar.gz" tarName="package.tar.gz"
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
script_dir="$(dirname $(readlink -f $0))" script_dir="$(dirname $(readlink -f $0))"
......
packaging/tools/makepkg.sh
浏览文件 @ 39eec5e8
...@@ -28,7 +28,7 @@ productName="TDengine" ...@@ -28,7 +28,7 @@ productName="TDengine"
serverName="taosd" serverName="taosd"
clientName="taos" clientName="taos"
configFile="taos.cfg" configFile="taos.cfg"
tarName="taos.tar.gz" tarName="package.tar.gz"
dumpName="taosdump" dumpName="taosdump"
benchmarkName="taosBenchmark" benchmarkName="taosBenchmark"
toolsName="taostools" toolsName="taostools"
...@@ -171,22 +171,22 @@ if [ -n "${taostools_bin_files}" ]; then ...@@ -171,22 +171,22 @@ if [ -n "${taostools_bin_files}" ]; then
&& cp ${taostools_bin_files} ${taostools_install_dir}/bin \ && cp ${taostools_bin_files} ${taostools_install_dir}/bin \
&& chmod a+x ${taostools_install_dir}/bin/* || : && chmod a+x ${taostools_install_dir}/bin/* || :
if [ -f ${top_dir}/tools/taos-tools/packaging/tools/install-taostools.sh ]; then if [ -f ${top_dir}/tools/taos-tools/packaging/tools/install-tools.sh ]; then
cp ${top_dir}/tools/taos-tools/packaging/tools/install-taostools.sh \ cp ${top_dir}/tools/taos-tools/packaging/tools/install-tools.sh \
${taostools_install_dir}/ > /dev/null \ ${taostools_install_dir}/ > /dev/null \
&& chmod a+x ${taostools_install_dir}/install-taostools.sh \ && chmod a+x ${taostools_install_dir}/install-tools.sh \
|| echo -e "failed to copy install-taostools.sh" || echo -e "failed to copy install-tools.sh"
else else
echo -e "install-taostools.sh not found" echo -e "install-tools.sh not found"
fi fi
if [ -f ${top_dir}/tools/taos-tools/packaging/tools/uninstall-taostools.sh ]; then if [ -f ${top_dir}/tools/taos-tools/packaging/tools/uninstall-tools.sh ]; then
cp ${top_dir}/tools/taos-tools/packaging/tools/uninstall-taostools.sh \ cp ${top_dir}/tools/taos-tools/packaging/tools/uninstall-tools.sh \
${taostools_install_dir}/ > /dev/null \ ${taostools_install_dir}/ > /dev/null \
&& chmod a+x ${taostools_install_dir}/uninstall-taostools.sh \ && chmod a+x ${taostools_install_dir}/uninstall-tools.sh \
|| echo -e "failed to copy uninstall-taostools.sh" || echo -e "failed to copy uninstall-tools.sh"
else else
echo -e "uninstall-taostools.sh not found" echo -e "uninstall-tools.sh not found"
fi fi
if [ -f ${build_dir}/lib/libavro.so.23.0.0 ]; then if [ -f ${build_dir}/lib/libavro.so.23.0.0 ]; then
......
packaging/tools/post.sh
浏览文件 @ 39eec5e8
...@@ -530,7 +530,7 @@ function install_service_on_sysvinit() { ...@@ -530,7 +530,7 @@ function install_service_on_sysvinit() {
function clean_service_on_systemd() { function clean_service_on_systemd() {
taosd_service_config="${service_config_dir}/taosd.service" taosd_service_config="${service_config_dir}/taosd.service"
# taosd service already is stoped before install in preinst script # taosd service already is stopped before install in preinst script
#if systemctl is-active --quiet taosd; then #if systemctl is-active --quiet taosd; then
# echo "TDengine is running, stopping it..." # echo "TDengine is running, stopping it..."
# ${csudo}systemctl stop taosd &> /dev/null || echo &> /dev/null # ${csudo}systemctl stop taosd &> /dev/null || echo &> /dev/null
......
packaging/tools/release_note
浏览文件 @ 39eec5e8
...@@ -72,7 +72,7 @@ New Features: ...@@ -72,7 +72,7 @@ New Features:
taos-1.4.13 (Released on 2018-12-14) taos-1.4.13 (Released on 2018-12-14)
Bugs Fixed: Bugs Fixed:
- Clients failed to connect to server due to unexpected and invalid packets recieved by the server. - Clients failed to connect to server due to unexpected and invalid packets received by the server.
Features Added: Features Added:
- Add support to HikariCP in TSDB JDBC driver. - Add support to HikariCP in TSDB JDBC driver.
......
packaging/tools/repair_link.sh
浏览文件 @ 39eec5e8
...@@ -8,7 +8,7 @@ read -p "Please enter link directory such as /var/lib/taos/tsdb: " linkDir ...@@ -8,7 +8,7 @@ read -p "Please enter link directory such as /var/lib/taos/tsdb: " linkDir
while true; do while true; do
if [ ! -d $linkDir ]; then if [ ! -d $linkDir ]; then
read -p "Paht not exists, please enter the correct link path:" linkDir read -p "Path not exists, please enter the correct link path:" linkDir
continue continue
fi fi
break break
...@@ -28,12 +28,12 @@ for linkFile in $(find -L $linkDir -xtype l); do ...@@ -28,12 +28,12 @@ for linkFile in $(find -L $linkDir -xtype l); do
if [ -z "${dirHash["$dirName"]}" ]; then if [ -z "${dirHash["$dirName"]}" ]; then
read -p "Please enter the directory to replace ${dirName}:" newDir read -p "Please enter the directory to replace ${dirName}:" newDir
read -p "Do you want to replcace all[y/N]?" replcaceAll read -p "Do you want to replace all[y/N]?" replaceAll
if [[ ( "${replcaceAll}" == "y") || ( "${replcaceAll}" == "Y") ]]; then if [[ ( "${replaceAll}" == "y") || ( "${replaceAll}" == "Y") ]]; then
dirHash["$dirName"]="$newDir" dirHash["$dirName"]="$newDir"
fi fi
fi fi
# Replcace the file # Replace the file
ln -sf "${newDir}/${baseName}" "${linkFile}" ln -sf "${newDir}/${baseName}" "${linkFile}"
done done
packaging/tools/tdengine.iss
浏览文件 @ 39eec5e8
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
[Setup] [Setup]
VersionInfoVersion={#MyAppVersion} VersionInfoVersion={#MyAppVersion}
AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB} AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB}
AppName={#MyAppName} AppName={#CusName}
AppVersion={#MyAppVersion} AppVersion={#MyAppVersion}
;AppVerName={#MyAppName} {#MyAppVersion} ;AppVerName={#MyAppName} {#MyAppVersion}
AppPublisher={#MyAppPublisher} AppPublisher={#MyAppPublisher}
...@@ -64,8 +64,8 @@ Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt ...@@ -64,8 +64,8 @@ Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt
[run] [run]
Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\{#CusName}\\taosd.exe --win_service""" ; Flags: runhidden Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden
Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\{#CusName}\\taosadapter.exe""" ; Flags: runhidden Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden
[UninstallRun] [UninstallRun]
RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden
......
packaging/tools/windows_before_install.txt
浏览文件 @ 39eec5e8
TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing.
TDengine will be installed under C:\TDengine, users can modify configuration file C:\TDengine\cfg\taos.cfg, set the log file path or other parameters. TDengine will be installed under C:\TDengine, users can modify configuration file C:\TDengine\cfg\taos.cfg, set the log file path or other parameters.
To start/stop TDengine with administrator privileges: sc start/stop taosd To start/stop TDengine with administrator privileges: sc start/stop taosd
......
source/client/src/clientRawBlockWrite.c
浏览文件 @ 39eec5e8
...@@ -194,6 +194,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) { ...@@ -194,6 +194,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) {
SDecoder coder; SDecoder coder;
char* string = NULL; char* string = NULL;
uDebug("processCreateStb called");
// decode and process req // decode and process req
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead);
...@@ -203,7 +204,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) { ...@@ -203,7 +204,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) {
goto _err; goto _err;
} }
string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE); string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE);
uDebug("processCreateStb %s", string);
_err: _err:
tDecoderClear(&coder); tDecoderClear(&coder);
return string; return string;
...@@ -213,6 +214,7 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) { ...@@ -213,6 +214,7 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) {
SVCreateStbReq req = {0}; SVCreateStbReq req = {0};
SDecoder coder; SDecoder coder;
char* string = NULL; char* string = NULL;
uDebug("processAlterStb called");
// decode and process req // decode and process req
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
...@@ -223,6 +225,7 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) { ...@@ -223,6 +225,7 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) {
goto _err; goto _err;
} }
string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen);
uDebug("processAlterStb %s", string);
_err: _err:
tDecoderClear(&coder); tDecoderClear(&coder);
...@@ -346,6 +349,7 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { ...@@ -346,6 +349,7 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
SVCreateTbReq* pCreateReq; SVCreateTbReq* pCreateReq;
char* string = NULL; char* string = NULL;
// decode // decode
uDebug("processCreateTable called");
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead);
tDecoderInit(&decoder, data, len); tDecoderInit(&decoder, data, len);
...@@ -359,9 +363,9 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { ...@@ -359,9 +363,9 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
if (pCreateReq->type == TSDB_CHILD_TABLE) { if (pCreateReq->type == TSDB_CHILD_TABLE) {
string = buildCreateCTableJson(req.pReqs, req.nReqs); string = buildCreateCTableJson(req.pReqs, req.nReqs);
} else if (pCreateReq->type == TSDB_NORMAL_TABLE) { } else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
string = string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE);
buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE);
} }
uDebug("processCreateTable :%s", string);
} }
_exit: _exit:
...@@ -377,6 +381,7 @@ _exit: ...@@ -377,6 +381,7 @@ _exit:
} }
static char* processAutoCreateTable(STaosxRsp* rsp) { static char* processAutoCreateTable(STaosxRsp* rsp) {
uDebug("processAutoCreateTable called");
if (rsp->createTableNum <= 0) { if (rsp->createTableNum <= 0) {
uError("WriteRaw:processAutoCreateTable rsp->createTableNum <= 0"); uError("WriteRaw:processAutoCreateTable rsp->createTableNum <= 0");
goto _exit; goto _exit;
...@@ -402,7 +407,7 @@ static char* processAutoCreateTable(STaosxRsp* rsp) { ...@@ -402,7 +407,7 @@ static char* processAutoCreateTable(STaosxRsp* rsp) {
} }
} }
string = buildCreateCTableJson(pCreateReq, rsp->createTableNum); string = buildCreateCTableJson(pCreateReq, rsp->createTableNum);
uDebug("processAutoCreateTable :%s", string);
_exit: _exit:
for (int i = 0; i < rsp->createTableNum; i++) { for (int i = 0; i < rsp->createTableNum; i++) {
tDecoderClear(&decoder[i]); tDecoderClear(&decoder[i]);
...@@ -422,6 +427,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { ...@@ -422,6 +427,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
char* string = NULL; char* string = NULL;
cJSON* json = NULL; cJSON* json = NULL;
uDebug("processAlterTable called");
// decode // decode
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead);
...@@ -527,6 +533,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { ...@@ -527,6 +533,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
break; break;
} }
string = cJSON_PrintUnformatted(json); string = cJSON_PrintUnformatted(json);
uDebug("processAlterTable :%s", string);
_exit: _exit:
cJSON_Delete(json); cJSON_Delete(json);
...@@ -539,6 +546,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { ...@@ -539,6 +546,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
SVDropStbReq req = {0}; SVDropStbReq req = {0};
char* string = NULL; char* string = NULL;
cJSON* json = NULL; cJSON* json = NULL;
uDebug("processDropSTable called");
// decode // decode
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
...@@ -560,7 +568,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { ...@@ -560,7 +568,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
cJSON_AddItemToObject(json, "tableName", tableName); cJSON_AddItemToObject(json, "tableName", tableName);
string = cJSON_PrintUnformatted(json); string = cJSON_PrintUnformatted(json);
uDebug("processDropSTable :%s", string);
_exit: _exit:
cJSON_Delete(json); cJSON_Delete(json);
tDecoderClear(&decoder); tDecoderClear(&decoder);
...@@ -573,6 +581,7 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) { ...@@ -573,6 +581,7 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) {
cJSON* json = NULL; cJSON* json = NULL;
char* string = NULL; char* string = NULL;
uDebug("processDeleteTable called");
// decode and process req // decode and process req
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead);
...@@ -599,7 +608,7 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) { ...@@ -599,7 +608,7 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) {
cJSON_AddItemToObject(json, "sql", sqlJson); cJSON_AddItemToObject(json, "sql", sqlJson);
string = cJSON_PrintUnformatted(json); string = cJSON_PrintUnformatted(json);
uDebug("processDeleteTable :%s", string);
_exit: _exit:
cJSON_Delete(json); cJSON_Delete(json);
tDecoderClear(&coder); tDecoderClear(&coder);
...@@ -612,6 +621,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { ...@@ -612,6 +621,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
char* string = NULL; char* string = NULL;
cJSON* json = NULL; cJSON* json = NULL;
uDebug("processDropTable called");
// decode // decode
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead);
...@@ -641,7 +651,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { ...@@ -641,7 +651,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
cJSON_AddItemToObject(json, "tableNameList", tableNameList); cJSON_AddItemToObject(json, "tableNameList", tableNameList);
string = cJSON_PrintUnformatted(json); string = cJSON_PrintUnformatted(json);
uDebug("processDropTable :%s", string);
_exit: _exit:
cJSON_Delete(json); cJSON_Delete(json);
tDecoderClear(&decoder); tDecoderClear(&decoder);
...@@ -655,6 +665,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -655,6 +665,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SRequestObj* pRequest = NULL; SRequestObj* pRequest = NULL;
uDebug("taosCreateStb called");
code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
...@@ -698,6 +709,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -698,6 +709,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
pReq.source = TD_REQ_FROM_TAOX; pReq.source = TD_REQ_FROM_TAOX;
pReq.igExists = true; pReq.igExists = true;
uDebug("taosCreateStb name:%s suid:%"PRId64" processSuid:%"PRId64, req.name, req.suid, pReq.suid);
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
SName tableName; SName tableName;
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
...@@ -744,6 +756,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -744,6 +756,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SRequestObj* pRequest = NULL; SRequestObj* pRequest = NULL;
uDebug("taosDropStb called");
code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
...@@ -763,11 +776,37 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -763,11 +776,37 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
goto end; goto end;
} }
SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
SName pName = {0};
toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
STableMeta *pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
code = TSDB_CODE_SUCCESS;
taosMemoryFreeClear(pTableMeta);
goto end;
}
if (code != TSDB_CODE_SUCCESS) {
uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", req.name);
goto end;
}
pReq.suid = pTableMeta->uid;
taosMemoryFreeClear(pTableMeta);
// build drop stable // build drop stable
pReq.igNotExists = true; pReq.igNotExists = true;
pReq.source = TD_REQ_FROM_TAOX; pReq.source = TD_REQ_FROM_TAOX;
pReq.suid = processSuid(req.suid, pRequest->pDb); // pReq.suid = processSuid(req.suid, pRequest->pDb);
uDebug("taosDropStb name:%s suid:%"PRId64" new suid:%"PRId64, req.name, req.suid, pReq.suid);
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
SName tableName = {0}; SName tableName = {0};
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
...@@ -792,7 +831,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -792,7 +831,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
launchQueryImpl(pRequest, &pQuery, true, NULL); launchQueryImpl(pRequest, &pQuery, true, NULL);
if (pRequest->code == TSDB_CODE_SUCCESS) { if (pRequest->code == TSDB_CODE_SUCCESS) {
SCatalog* pCatalog = NULL; // SCatalog* pCatalog = NULL;
catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
catalogRemoveTableMeta(pCatalog, &tableName); catalogRemoveTableMeta(pCatalog, &tableName);
} }
...@@ -825,6 +864,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -825,6 +864,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
SQuery* pQuery = NULL; SQuery* pQuery = NULL;
SHashObj* pVgroupHashmap = NULL; SHashObj* pVgroupHashmap = NULL;
uDebug("taosCreateTable called");
code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
...@@ -884,13 +924,16 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -884,13 +924,16 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
if (pCreateReq->type == TSDB_CHILD_TABLE) { if (pCreateReq->type == TSDB_CHILD_TABLE) {
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
SName sName = {0}; SName sName = {0};
pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); tb_uid_t oldSuid = pCreateReq->ctb.suid;
// pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName);
code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta); code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", pCreateReq->ctb.stbName); uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", pCreateReq->ctb.stbName);
goto end; goto end;
} }
pCreateReq->ctb.suid = pTableMeta->uid;
uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" new suid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid);
for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) { for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) {
char* tName = taosArrayGet(pCreateReq->ctb.tagName, i); char* tName = taosArrayGet(pCreateReq->ctb.tagName, i);
...@@ -979,6 +1022,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -979,6 +1022,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
SQuery* pQuery = NULL; SQuery* pQuery = NULL;
SHashObj* pVgroupHashmap = NULL; SHashObj* pVgroupHashmap = NULL;
uDebug("taosDropTable called");
code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
...@@ -1022,7 +1066,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -1022,7 +1066,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pDropReq = req.pReqs + iReq; pDropReq = req.pReqs + iReq;
pDropReq->igNotExists = true; pDropReq->igNotExists = true;
pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb); // pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
SVgroupInfo pInfo = {0}; SVgroupInfo pInfo = {0};
SName pName = {0}; SName pName = {0};
...@@ -1032,6 +1076,22 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -1032,6 +1076,22 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
goto end; goto end;
} }
STableMeta *pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
code = TSDB_CODE_SUCCESS;
taosMemoryFreeClear(pTableMeta);
continue;
}
if (code != TSDB_CODE_SUCCESS) {
uError("taosDropTable:catalogGetTableMeta failed. table name: %s", pDropReq->name);
goto end;
}
tb_uid_t oldSuid = pDropReq->suid;
pDropReq->suid = pTableMeta->suid;
taosMemoryFreeClear(pTableMeta);
uDebug("taosDropTable name:%s suid:%"PRId64" new suid:%"PRId64, pDropReq->name, oldSuid, pDropReq->suid);
taosArrayPush(pRequest->tableList, &pName); taosArrayPush(pRequest->tableList, &pName);
SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
if (pTableBatch == NULL) { if (pTableBatch == NULL) {
...@@ -1046,6 +1106,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -1046,6 +1106,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
} }
} }
if (taosHashGetSize(pVgroupHashmap) == 0){
goto end;
}
SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap);
if (NULL == pBufArray) { if (NULL == pBufArray) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
...@@ -1114,6 +1177,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -1114,6 +1177,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
SDecoder coder = {0}; SDecoder coder = {0};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
uDebug("taosDeleteData called");
// decode and process req // decode and process req
void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); void* data = POINTER_SHIFT(meta, sizeof(SMsgHead));
int32_t len = metaLen - sizeof(SMsgHead); int32_t len = metaLen - sizeof(SMsgHead);
...@@ -1151,6 +1215,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -1151,6 +1215,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
SArray* pArray = NULL; SArray* pArray = NULL;
SVgDataBlocks* pVgData = NULL; SVgDataBlocks* pVgData = NULL;
uDebug("taosAlterTable called");
code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
...@@ -1196,6 +1261,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { ...@@ -1196,6 +1261,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
goto end; goto end;
} }
uDebug("taosAlterTable name:%s", req.tbName);
pArray = taosArrayInit(1, sizeof(void*)); pArray = taosArrayInit(1, sizeof(void*));
if (NULL == pArray) { if (NULL == pArray) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
...@@ -1261,6 +1327,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch ...@@ -1261,6 +1327,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
SQuery* pQuery = NULL; SQuery* pQuery = NULL;
SHashObj* pVgHash = NULL; SHashObj* pVgHash = NULL;
uDebug("taos_write_raw_block_with_fields called");
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
if (!pRequest) { if (!pRequest) {
...@@ -1280,6 +1347,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch ...@@ -1280,6 +1347,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname));
tstrncpy(pName.tname, tbname, sizeof(pName.tname)); tstrncpy(pName.tname, tbname, sizeof(pName.tname));
uDebug("taos_write_raw_block_with_fields name:%s", tbname);
struct SCatalog* pCatalog = NULL; struct SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
...@@ -1342,6 +1410,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) ...@@ -1342,6 +1410,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
SQuery* pQuery = NULL; SQuery* pQuery = NULL;
SHashObj* pVgHash = NULL; SHashObj* pVgHash = NULL;
uDebug("taos_write_raw_block called");
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
if (!pRequest) { if (!pRequest) {
uError("WriteRaw:createRequest error request is null"); uError("WriteRaw:createRequest error request is null");
...@@ -1360,6 +1429,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) ...@@ -1360,6 +1429,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname));
tstrncpy(pName.tname, tbname, sizeof(pName.tname)); tstrncpy(pName.tname, tbname, sizeof(pName.tname));
uDebug("taos_write_raw_block name:%s", tbname);
struct SCatalog* pCatalog = NULL; struct SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
...@@ -1423,6 +1493,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { ...@@ -1423,6 +1493,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
SMqRspObj rspObj = {0}; SMqRspObj rspObj = {0};
SDecoder decoder = {0}; SDecoder decoder = {0};
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
uDebug("tmqWriteRawDataImpl called");
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
...@@ -1468,7 +1539,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { ...@@ -1468,7 +1539,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); uDebug("tmqWriteRawDataImpl raw data block num:%d", rspObj.rsp.blockNum);
while (++rspObj.resIter < rspObj.rsp.blockNum) { while (++rspObj.resIter < rspObj.rsp.blockNum) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter);
if (!rspObj.rsp.withSchema) { if (!rspObj.rsp.withSchema) {
...@@ -1483,7 +1554,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { ...@@ -1483,7 +1554,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
goto end; goto end;
} }
uDebug("raw data tbname:%s\n", tbName); uDebug("tmqWriteRawDataImpl raw data tbname:%s", tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
strcpy(pName.dbname, pRequest->pDb); strcpy(pName.dbname, pRequest->pDb);
strcpy(pName.tname, tbName); strcpy(pName.tname, tbName);
...@@ -1556,6 +1627,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) ...@@ -1556,6 +1627,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
SDecoder decoder = {0}; SDecoder decoder = {0};
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
SVCreateTbReq* pCreateReqDst = NULL; SVCreateTbReq* pCreateReqDst = NULL;
uDebug("tmqWriteRawMetaDataImpl called");
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
...@@ -1602,7 +1674,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) ...@@ -1602,7 +1674,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); uDebug("tmqWriteRawMetaDataImpl raw data block num:%d", rspObj.rsp.blockNum);
while (++rspObj.resIter < rspObj.rsp.blockNum) { while (++rspObj.resIter < rspObj.rsp.blockNum) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter);
if (!rspObj.rsp.withSchema) { if (!rspObj.rsp.withSchema) {
...@@ -1617,7 +1689,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) ...@@ -1617,7 +1689,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
goto end; goto end;
} }
uDebug("raw data tbname:%s\n", tbName); uDebug("tmqWriteRawMetaDataImpl raw data tbname:%s\n", tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
strcpy(pName.dbname, pRequest->pDb); strcpy(pName.dbname, pRequest->pDb);
strcpy(pName.tname, tbName); strcpy(pName.tname, tbName);
...@@ -1644,7 +1716,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) ...@@ -1644,7 +1716,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
} }
if (strcmp(tbName, pCreateReq.name) == 0) { if (strcmp(tbName, pCreateReq.name) == 0) {
cloneSVreateTbReq(&pCreateReq, &pCreateReqDst); cloneSVreateTbReq(&pCreateReq, &pCreateReqDst);
pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb); // pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
tDecoderClear(&decoderTmp); tDecoderClear(&decoderTmp);
break; break;
} }
...@@ -1677,6 +1749,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) ...@@ -1677,6 +1749,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
if (pCreateReqDst) { if (pCreateReqDst) {
pTableMeta->vgId = vg.vgId; pTableMeta->vgId = vg.vgId;
pTableMeta->uid = pCreateReqDst->uid; pTableMeta->uid = pCreateReqDst->uid;
pCreateReqDst->ctb.suid = pTableMeta->suid;
} }
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
if (hData == NULL) { if (hData == NULL) {
...@@ -1726,6 +1799,7 @@ end: ...@@ -1726,6 +1799,7 @@ end:
} }
char* tmq_get_json_meta(TAOS_RES* res) { char* tmq_get_json_meta(TAOS_RES* res) {
uDebug("tmq_get_json_meta called");
if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) { if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) {
return NULL; return NULL;
} }
...@@ -1760,6 +1834,7 @@ char* tmq_get_json_meta(TAOS_RES* res) { ...@@ -1760,6 +1834,7 @@ char* tmq_get_json_meta(TAOS_RES* res) {
void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); }
int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
uDebug("tmq_get_raw called");
if (!raw || !res) { if (!raw || !res) {
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
...@@ -1768,6 +1843,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { ...@@ -1768,6 +1843,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
raw->raw = pMetaRspObj->metaRsp.metaRsp; raw->raw = pMetaRspObj->metaRsp.metaRsp;
raw->raw_len = pMetaRspObj->metaRsp.metaRspLen; raw->raw_len = pMetaRspObj->metaRsp.metaRspLen;
raw->raw_type = pMetaRspObj->metaRsp.resMsgType; raw->raw_type = pMetaRspObj->metaRsp.resMsgType;
uDebug("tmq_get_raw meta");
} else if (TD_RES_TMQ(res)) { } else if (TD_RES_TMQ(res)) {
SMqRspObj* rspObj = ((SMqRspObj*)res); SMqRspObj* rspObj = ((SMqRspObj*)res);
...@@ -1787,6 +1863,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { ...@@ -1787,6 +1863,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
raw->raw = buf; raw->raw = buf;
raw->raw_len = len; raw->raw_len = len;
raw->raw_type = RES_TYPE__TMQ; raw->raw_type = RES_TYPE__TMQ;
uDebug("tmq_get_raw data");
} else if (TD_RES_TMQ_METADATA(res)) { } else if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res); SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res);
...@@ -1806,19 +1883,23 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { ...@@ -1806,19 +1883,23 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
raw->raw = buf; raw->raw = buf;
raw->raw_len = len; raw->raw_len = len;
raw->raw_type = RES_TYPE__TMQ_METADATA; raw->raw_type = RES_TYPE__TMQ_METADATA;
uDebug("tmq_get_raw meta data");
} else { } else {
uError("tmq_get_raw error:%d", *(int8_t*)res);
return TSDB_CODE_TMQ_INVALID_MSG; return TSDB_CODE_TMQ_INVALID_MSG;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void tmq_free_raw(tmq_raw_data raw) { void tmq_free_raw(tmq_raw_data raw) {
uDebug("tmq_free_raw raw_type:%d", raw.raw_type);
if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) { if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) {
taosMemoryFree(raw.raw); taosMemoryFree(raw.raw);
} }
} }
int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
uDebug("tmq_write_raw called");
if (!taos) { if (!taos) {
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
......
source/client/src/clientTmq.c
浏览文件 @ 39eec5e8
此差异已折叠。
source/client/test/clientTests.cpp
浏览文件 @ 39eec5e8
...@@ -898,7 +898,86 @@ TEST(clientCase, update_test) { ...@@ -898,7 +898,86 @@ TEST(clientCase, update_test) {
} }
} }
TEST(clientCase, subscription_test) { TEST(clientCase, sub_db_test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
// TAOS_RES* pRes = taos_query(pConn, "create topic topic_t1 as select * from t1");
// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
// printf("failed to create topic, code:%s", taos_errstr(pRes));
// taos_free_result(pRes);
// return;
// }
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
tmq_conf_set(conf, "group.id", "cgrpNamedb");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "auto.offset.reset", "earliest");
tmq_conf_set(conf, "experimental.snapshot.enable", "true");
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
tmq_conf_destroy(conf);
// 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new();
tmq_list_append(topicList, "topic_db1");
// 启动订阅
tmq_subscribe(tmq, topicList);
tmq_list_destroy(topicList);
TAOS_FIELD* fields = NULL;
int32_t numOfFields = 0;
int32_t precision = 0;
int32_t totalRows = 0;
int32_t msgCnt = 0;
int32_t timeout = 5000;
int32_t count = 0;
while (1) {
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
if (pRes) {
char buf[1024];
int32_t rows = 0;
const char* topicName = tmq_get_topic_name(pRes);
const char* dbName = tmq_get_db_name(pRes);
int32_t vgroupId = tmq_get_vgroup_id(pRes);
printf("topic: %s\n", topicName);
printf("db: %s\n", dbName);
printf("vgroup id: %d\n", vgroupId);
if (count ++ > 200) {
tmq_unsubscribe(tmq);
break;
}
while (1) {
TAOS_ROW row = taos_fetch_row(pRes);
if (row == NULL) break;
fields = taos_fetch_fields(pRes);
numOfFields = taos_field_count(pRes);
precision = taos_result_precision(pRes);
rows++;
taos_print_row(buf, row, fields, numOfFields);
printf("precision: %d, row content: %s\n", precision, buf);
}
}
// return rows;
}
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
}
TEST(clientCase, sub_tb_test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
...@@ -925,7 +1004,7 @@ TEST(clientCase, subscription_test) { ...@@ -925,7 +1004,7 @@ TEST(clientCase, subscription_test) {
// 创建订阅 topics 列表 // 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new(); tmq_list_t* topicList = tmq_list_new();
// tmq_list_append(topicList, "topic_t1"); tmq_list_append(topicList, "topic_t1");
// 启动订阅 // 启动订阅
tmq_subscribe(tmq, topicList); tmq_subscribe(tmq, topicList);
...@@ -954,7 +1033,7 @@ TEST(clientCase, subscription_test) { ...@@ -954,7 +1033,7 @@ TEST(clientCase, subscription_test) {
printf("db: %s\n", dbName); printf("db: %s\n", dbName);
printf("vgroup id: %d\n", vgroupId); printf("vgroup id: %d\n", vgroupId);
if (count ++ > 20) { if (count ++ > 200) {
tmq_unsubscribe(tmq); tmq_unsubscribe(tmq);
break; break;
} }
......
source/common/src/systable.c
浏览文件 @ 39eec5e8
...@@ -233,6 +233,7 @@ static const SSysDbTableSchema vgroupsSchema[] = { ...@@ -233,6 +233,7 @@ static const SSysDbTableSchema vgroupsSchema[] = {
{.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "cacheTables", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
// {.name = "compact_start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, // {.name = "compact_start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
}; };
......
source/common/src/tglobal.c
浏览文件 @ 39eec5e8
...@@ -154,6 +154,7 @@ char tsTagFilterCache = 0; ...@@ -154,6 +154,7 @@ char tsTagFilterCache = 0;
// positive value (in MB) // positive value (in MB)
int32_t tsQueryBufferSize = -1; int32_t tsQueryBufferSize = -1;
int64_t tsQueryBufferSizeBytes = -1; int64_t tsQueryBufferSizeBytes = -1;
int32_t tsCacheLazyLoadThreshold = 500;
int32_t tsDiskCfgNum = 0; int32_t tsDiskCfgNum = 0;
SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0};
...@@ -497,6 +498,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { ...@@ -497,6 +498,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "disableStream", tsDisableStream, 0) != 0) return -1; if (cfgAddBool(pCfg, "disableStream", tsDisableStream, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1;
GRANT_CFG_ADD; GRANT_CFG_ADD;
return 0; return 0;
} }
...@@ -824,6 +827,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { ...@@ -824,6 +827,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
} }
tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval; tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
GRANT_CFG_GET; GRANT_CFG_GET;
......
source/common/src/tmsg.c
浏览文件 @ 39eec5e8
...@@ -1070,7 +1070,8 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { ...@@ -1070,7 +1070,8 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1; if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1;
if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1; if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1;
if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1; if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1;
if (tEncodeI64(&encoder, reserved) < 0) return -1; if (tEncodeI32(&encoder, pload->numOfCachedTables) < 0) return -1;
if (tEncodeI32(&encoder, reserved) < 0) return -1;
if (tEncodeI64(&encoder, reserved) < 0) return -1; if (tEncodeI64(&encoder, reserved) < 0) return -1;
if (tEncodeI64(&encoder, reserved) < 0) return -1; if (tEncodeI64(&encoder, reserved) < 0) return -1;
} }
...@@ -1148,7 +1149,8 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { ...@@ -1148,7 +1149,8 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1; if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1;
if (tDecodeI64(&decoder, &reserved) < 0) return -1; if (tDecodeI32(&decoder, &vload.numOfCachedTables) < 0) return -1;
if (tDecodeI32(&decoder, (int32_t*)&reserved) < 0) return -1;
if (tDecodeI64(&decoder, &reserved) < 0) return -1; if (tDecodeI64(&decoder, &reserved) < 0) return -1;
if (tDecodeI64(&decoder, &reserved) < 0) return -1; if (tDecodeI64(&decoder, &reserved) < 0) return -1;
if (taosArrayPush(pReq->pVloads, &vload) == NULL) { if (taosArrayPush(pReq->pVloads, &vload) == NULL) {
...@@ -6637,8 +6639,9 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { ...@@ -6637,8 +6639,9 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
} else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) { } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
snprintf(buf, maxLen, "offset(snapshot) uid:%" PRId64 " ts:%" PRId64, pVal->uid, pVal->ts); snprintf(buf, maxLen, "offset(snapshot) uid:%" PRId64 " ts:%" PRId64, pVal->uid, pVal->ts);
} else { } else {
ASSERT(0); return TSDB_CODE_INVALID_PARA;
} }
return 0; return 0;
} }
...@@ -6821,8 +6824,7 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { ...@@ -6821,8 +6824,7 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
} }
void tDeleteSMqDataRsp(SMqDataRsp *pRsp) { void tDeleteSMqDataRsp(SMqDataRsp *pRsp) {
taosArrayDestroy(pRsp->blockDataLen); pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);;
pRsp->blockDataLen = NULL;
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL; pRsp->blockData = NULL;
taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper); taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);
......
source/dnode/mnode/impl/inc/mndConsumer.h
浏览文件 @ 39eec5e8
...@@ -24,10 +24,10 @@ extern "C" { ...@@ -24,10 +24,10 @@ extern "C" {
enum { enum {
MQ_CONSUMER_STATUS__MODIFY = 1, MQ_CONSUMER_STATUS__MODIFY = 1,
MQ_CONSUMER_STATUS__MODIFY_IN_REB, MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore
MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__READY,
MQ_CONSUMER_STATUS__LOST, MQ_CONSUMER_STATUS__LOST,
MQ_CONSUMER_STATUS__LOST_IN_REB, MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore
MQ_CONSUMER_STATUS__LOST_REBD, MQ_CONSUMER_STATUS__LOST_REBD,
MQ_CONSUMER_STATUS__REMOVED, MQ_CONSUMER_STATUS__REMOVED,
}; };
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ 39eec5e8
...@@ -359,6 +359,7 @@ typedef struct { ...@@ -359,6 +359,7 @@ typedef struct {
int8_t replica; int8_t replica;
SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
void* pTsma; void* pTsma;
int32_t numOfCachedTables;
} SVgObj; } SVgObj;
typedef struct { typedef struct {
......
source/dnode/mnode/impl/inc/mndInt.h
浏览文件 @ 39eec5e8
...@@ -113,6 +113,7 @@ typedef struct SMnode { ...@@ -113,6 +113,7 @@ typedef struct SMnode {
bool deploy; bool deploy;
char *path; char *path;
int64_t checkTime; int64_t checkTime;
SyncIndex applied;
SSdb *pSdb; SSdb *pSdb;
SArray *pSteps; SArray *pSteps;
SQHandle *pQuery; SQHandle *pQuery;
......
source/dnode/mnode/impl/src/mndConsumer.c
浏览文件 @ 39eec5e8
...@@ -77,7 +77,7 @@ void mndCleanupConsumer(SMnode *pMnode) {} ...@@ -77,7 +77,7 @@ void mndCleanupConsumer(SMnode *pMnode) {}
bool mndRebTryStart() { bool mndRebTryStart() {
int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1);
mInfo("tq timer, rebalance counter old val:%d", old); mDebug("tq timer, rebalance counter old val:%d", old);
return old == 0; return old == 0;
} }
...@@ -253,11 +253,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { ...@@ -253,11 +253,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer; SMqConsumerObj *pConsumer;
void *pIter = NULL; void *pIter = NULL;
mTrace("start to process mq timer"); mDebug("start to process mq timer");
// rebalance cannot be parallel // rebalance cannot be parallel
if (!mndRebTryStart()) { if (!mndRebTryStart()) {
mInfo("mq rebalance already in progress, do nothing"); mDebug("mq rebalance already in progress, do nothing");
return 0; return 0;
} }
...@@ -317,7 +317,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { ...@@ -317,7 +317,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
} }
taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pConsumer->lock);
} else if (status == MQ_CONSUMER_STATUS__MODIFY) { } else if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
taosRLockLatch(&pConsumer->lock); taosRLockLatch(&pConsumer->lock);
int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
...@@ -356,7 +356,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { ...@@ -356,7 +356,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
} else { } else {
taosHashCleanup(pRebMsg->rebSubHash); taosHashCleanup(pRebMsg->rebSubHash);
rpcFreeCont(pRebMsg); rpcFreeCont(pRebMsg);
mInfo("mq rebalance finished, no modification"); mDebug("mq rebalance finished, no modification");
mndRebEnd(); mndRebEnd();
} }
return 0; return 0;
......
source/dnode/mnode/impl/src/mndDb.c
浏览文件 @ 39eec5e8
...@@ -279,6 +279,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { ...@@ -279,6 +279,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.cacheLast = pNew->cfg.cacheLast; pOld->cfg.cacheLast = pNew->cfg.cacheLast;
pOld->cfg.replications = pNew->cfg.replications; pOld->cfg.replications = pNew->cfg.replications;
pOld->cfg.sstTrigger = pNew->cfg.sstTrigger; pOld->cfg.sstTrigger = pNew->cfg.sstTrigger;
pOld->cfg.minRows = pNew->cfg.minRows;
pOld->cfg.maxRows = pNew->cfg.maxRows;
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
pOld->compactStartTime = pNew->compactStartTime; pOld->compactStartTime = pNew->compactStartTime;
taosWUnLockLatch(&pOld->lock); taosWUnLockLatch(&pOld->lock);
...@@ -616,13 +618,8 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { ...@@ -616,13 +618,8 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
} }
} else { } else {
if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { if (terrno == TSDB_CODE_MND_DB_IN_CREATING) {
if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { code = terrno;
mInfo("db:%s, is creating and createdb response after trans finished", createReq.db);
code = TSDB_CODE_ACTION_IN_PROGRESS;
goto _OVER;
} else {
goto _OVER; goto _OVER;
}
} else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) { } else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) {
goto _OVER; goto _OVER;
} else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) { } else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) {
...@@ -1270,14 +1267,9 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { ...@@ -1270,14 +1267,9 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) {
usedbRsp.errCode = terrno; usedbRsp.errCode = terrno;
if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { if (terrno == TSDB_CODE_MND_DB_IN_CREATING) {
if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) { code = terrno;
mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db);
code = TSDB_CODE_ACTION_IN_PROGRESS;
goto _OVER; goto _OVER;
} }
}
mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr());
} else { } else {
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) {
goto _OVER; goto _OVER;
......
source/dnode/mnode/impl/src/mndDnode.c
浏览文件 @ 39eec5e8
...@@ -412,6 +412,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { ...@@ -412,6 +412,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
if (pVgroup != NULL) { if (pVgroup != NULL) {
if (pVload->syncState == TAOS_SYNC_STATE_LEADER) { if (pVload->syncState == TAOS_SYNC_STATE_LEADER) {
pVgroup->cacheUsage = pVload->cacheUsage; pVgroup->cacheUsage = pVload->cacheUsage;
pVgroup->numOfCachedTables = pVload->numOfCachedTables;
pVgroup->numOfTables = pVload->numOfTables; pVgroup->numOfTables = pVload->numOfTables;
pVgroup->numOfTimeSeries = pVload->numOfTimeSeries; pVgroup->numOfTimeSeries = pVload->numOfTimeSeries;
pVgroup->totalStorage = pVload->totalStorage; pVgroup->totalStorage = pVload->totalStorage;
...@@ -440,7 +441,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { ...@@ -440,7 +441,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
if (roleChanged) { if (roleChanged) {
SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
if (pDb != NULL && pDb->stateTs != curMs) { if (pDb != NULL && pDb->stateTs != curMs) {
mInfo("db:%s, stateTs changed by status msg, old stateTs:%" PRId64 " new stateTs:%" PRId64, pDb->name, pDb->stateTs, curMs); mInfo("db:%s, stateTs changed by status msg, old stateTs:%" PRId64 " new stateTs:%" PRId64, pDb->name,
pDb->stateTs, curMs);
pDb->stateTs = curMs; pDb->stateTs = curMs;
} }
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
......
source/dnode/mnode/impl/src/mndMain.c
浏览文件 @ 39eec5e8
...@@ -380,11 +380,13 @@ static int32_t mndInitSdb(SMnode *pMnode) { ...@@ -380,11 +380,13 @@ static int32_t mndInitSdb(SMnode *pMnode) {
} }
static int32_t mndOpenSdb(SMnode *pMnode) { static int32_t mndOpenSdb(SMnode *pMnode) {
int32_t code = 0;
if (!pMnode->deploy) { if (!pMnode->deploy) {
return sdbReadFile(pMnode->pSdb); code = sdbReadFile(pMnode->pSdb);
} else {
return 0;
} }
atomic_store_64(&pMnode->applied, pMnode->pSdb->commitIndex);
return code;
} }
static void mndCleanupSdb(SMnode *pMnode) { static void mndCleanupSdb(SMnode *pMnode) {
......
source/dnode/mnode/impl/src/mndSubscribe.c
浏览文件 @ 39eec5e8
...@@ -224,7 +224,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ...@@ -224,7 +224,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, sub, pVgEp->vgId, consumerId); mInfo("sub:%s mq re-balance remove vgId:%d from consumer:0x%" PRIx64, sub, pVgEp->vgId, consumerId);
} }
taosArrayDestroy(pConsumerEp->vgs); taosArrayDestroy(pConsumerEp->vgs);
taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
...@@ -329,7 +329,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ...@@ -329,7 +329,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
taosArrayPush(pOutput->newConsumers, &consumerId); taosArrayPush(pOutput->newConsumers, &consumerId);
mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, sub, consumerId); mInfo("sub:%s mq rebalance add new consumer:0x%" PRIx64, sub, consumerId);
} }
} }
...@@ -357,7 +357,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ...@@ -357,7 +357,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pConsumerEp->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pOutput->rebVgs, pRebVg);
mInfo("mq rebalance: add vgId:%d to consumer:%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId, mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId); pConsumerEp->consumerId);
} }
} }
...@@ -387,12 +387,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ...@@ -387,12 +387,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pConsumerEp->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
if (pRebVg->newConsumerId == pRebVg->oldConsumerId) { if (pRebVg->newConsumerId == pRebVg->oldConsumerId) {
mInfo("mq rebalance: skip vg %d for same consumer:%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId, mInfo("mq rebalance: skip vg %d for same consumer:0x%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId); pConsumerEp->consumerId);
continue; continue;
} }
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pOutput->rebVgs, pRebVg);
mInfo("mq rebalance: add vgId:%d to consumer:%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId, mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId); pConsumerEp->consumerId);
} }
} else { } else {
...@@ -1019,7 +1019,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock ...@@ -1019,7 +1019,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false);
mDebug("mnd show subscriptions: topic %s, consumer:%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), mDebug("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic),
pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId); pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId);
// offset // offset
......
source/dnode/mnode/impl/src/mndSync.c
浏览文件 @ 39eec5e8
...@@ -129,6 +129,14 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta ...@@ -129,6 +129,14 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta
int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
int32_t code = 0; int32_t code = 0;
pMsg->info.conn.applyIndex = pMeta->index;
pMsg->info.conn.applyTerm = pMeta->term;
if (pMsg->code == 0) {
SMnode *pMnode = pFsm->data;
atomic_store_64(&pMnode->applied, pMsg->info.conn.applyIndex);
}
if (!syncUtilUserCommit(pMsg->msgType)) { if (!syncUtilUserCommit(pMsg->msgType)) {
goto _out; goto _out;
} }
...@@ -140,6 +148,11 @@ _out: ...@@ -140,6 +148,11 @@ _out:
return code; return code;
} }
SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) {
SMnode *pMnode = pFSM->data;
return atomic_load_64(&pMnode->applied);
}
int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) { int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) {
mInfo("start to read snapshot from sdb in atomic way"); mInfo("start to read snapshot from sdb in atomic way");
SMnode *pMnode = pFsm->data; SMnode *pMnode = pFsm->data;
...@@ -153,7 +166,7 @@ static void mndSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) { ...@@ -153,7 +166,7 @@ static void mndSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) {
sdbGetCommitInfo(pMnode->pSdb, &pSnapshot->lastApplyIndex, &pSnapshot->lastApplyTerm, &pSnapshot->lastConfigIndex); sdbGetCommitInfo(pMnode->pSdb, &pSnapshot->lastApplyIndex, &pSnapshot->lastApplyTerm, &pSnapshot->lastConfigIndex);
} }
void mndRestoreFinish(const SSyncFSM *pFsm) { void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) {
SMnode *pMnode = pFsm->data; SMnode *pMnode = pFsm->data;
if (!pMnode->deploy) { if (!pMnode->deploy) {
...@@ -167,6 +180,8 @@ void mndRestoreFinish(const SSyncFSM *pFsm) { ...@@ -167,6 +180,8 @@ void mndRestoreFinish(const SSyncFSM *pFsm) {
} else { } else {
mInfo("vgId:1, sync restore finished"); mInfo("vgId:1, sync restore finished");
} }
ASSERT(commitIdx == mndSyncAppliedIndex(pFsm));
} }
int32_t mndSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) { int32_t mndSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) {
...@@ -253,6 +268,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { ...@@ -253,6 +268,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) {
SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
pFsm->data = pMnode; pFsm->data = pMnode;
pFsm->FpCommitCb = mndSyncCommitMsg; pFsm->FpCommitCb = mndSyncCommitMsg;
pFsm->FpAppliedIndexCb = mndSyncAppliedIndex;
pFsm->FpPreCommitCb = NULL; pFsm->FpPreCommitCb = NULL;
pFsm->FpRollBackCb = NULL; pFsm->FpRollBackCb = NULL;
pFsm->FpRestoreFinishCb = mndRestoreFinish; pFsm->FpRestoreFinishCb = mndRestoreFinish;
......
source/dnode/mnode/impl/src/mndVgroup.c
浏览文件 @ 39eec5e8
...@@ -803,6 +803,9 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p ...@@ -803,6 +803,9 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
int32_t cacheUsage = (int32_t)pVgroup->cacheUsage; int32_t cacheUsage = (int32_t)pVgroup->cacheUsage;
colDataSetVal(pColInfo, numOfRows, (const char *)&cacheUsage, false); colDataSetVal(pColInfo, numOfRows, (const char *)&cacheUsage, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->numOfCachedTables, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false);
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ 39eec5e8
...@@ -201,6 +201,7 @@ int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int6 ...@@ -201,6 +201,7 @@ int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int6
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode); size_t tsdbCacheGetCapacity(SVnode *pVnode);
size_t tsdbCacheGetUsage(SVnode *pVnode); size_t tsdbCacheGetUsage(SVnode *pVnode);
int32_t tsdbCacheGetElems(SVnode *pVnode);
// tq // tq
typedef struct SMetaTableInfo { typedef struct SMetaTableInfo {
...@@ -264,7 +265,7 @@ int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); ...@@ -264,7 +265,7 @@ int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char* id); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret);
int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
......
source/dnode/vnode/src/inc/tq.h
浏览文件 @ 39eec5e8
...@@ -80,7 +80,6 @@ typedef struct { ...@@ -80,7 +80,6 @@ typedef struct {
typedef struct { typedef struct {
int8_t subType; int8_t subType;
STqReader* pExecReader; STqReader* pExecReader;
qTaskInfo_t task; qTaskInfo_t task;
union { union {
...@@ -92,30 +91,21 @@ typedef struct { ...@@ -92,30 +91,21 @@ typedef struct {
} STqExecHandle; } STqExecHandle;
typedef struct { typedef struct {
// info
char subKey[TSDB_SUBSCRIBE_KEY_LEN]; char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId; int64_t consumerId;
int32_t epoch; int32_t epoch;
int8_t fetchMeta; int8_t fetchMeta;
int64_t snapshotVer; int64_t snapshotVer;
SWalReader* pWalReader; SWalReader* pWalReader;
SWalRef* pRef; SWalRef* pRef;
STqPushHandle pushHandle; // push
// push STqExecHandle execHandle; // exec
STqPushHandle pushHandle;
// exec
STqExecHandle execHandle;
} STqHandle; } STqHandle;
typedef struct { typedef struct {
SMqDataRsp dataRsp; SMqDataRsp* pDataRsp;
char subKey[TSDB_SUBSCRIBE_KEY_LEN]; char subKey[TSDB_SUBSCRIBE_KEY_LEN];
SRpcHandleInfo pInfo; SRpcHandleInfo info;
} STqPushEntry; } STqPushEntry;
struct STQ { struct STQ {
...@@ -151,13 +141,13 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); ...@@ -151,13 +141,13 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
// tqRead // tqRead
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset); int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset);
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
// tqExec // tqExec
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp); int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp);
// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); // int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp);
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type);
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry);
// tqMeta // tqMeta
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ 39eec5e8
...@@ -706,6 +706,7 @@ typedef struct SMergeTree { ...@@ -706,6 +706,7 @@ typedef struct SMergeTree {
bool destroyLoadInfo; bool destroyLoadInfo;
SSttBlockLoadInfo *pLoadInfo; SSttBlockLoadInfo *pLoadInfo;
const char *idStr; const char *idStr;
bool ignoreEarlierTs;
} SMergeTree; } SMergeTree;
typedef struct { typedef struct {
...@@ -748,9 +749,10 @@ struct SDiskDataBuilder { ...@@ -748,9 +749,10 @@ struct SDiskDataBuilder {
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
bool destroyLoadInfo, const char *idStr); bool destroyLoadInfo, const char *idStr, bool strictTimeRange);
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
bool tMergeTreeNext(SMergeTree *pMTree); bool tMergeTreeNext(SMergeTree *pMTree);
bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree);
......
source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @ 39eec5e8
...@@ -192,6 +192,9 @@ void tqCleanUp(); ...@@ -192,6 +192,9 @@ void tqCleanUp();
STQ* tqOpen(const char* path, SVnode* pVnode); STQ* tqOpen(const char* path, SVnode* pVnode);
void tqClose(STQ*); void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type);
int tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
int tqCommit(STQ*); int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId);
......
source/dnode/vnode/src/meta/metaCache.c
浏览文件 @ 39eec5e8
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
*/ */
#include "meta.h" #include "meta.h"
#define TAG_FILTER_RES_KEY_LEN 32
#define META_CACHE_BASE_BUCKET 1024 #define META_CACHE_BASE_BUCKET 1024
#define META_CACHE_STATS_BUCKET 16 #define META_CACHE_STATS_BUCKET 16
...@@ -34,7 +35,6 @@ typedef struct SMetaStbStatsEntry { ...@@ -34,7 +35,6 @@ typedef struct SMetaStbStatsEntry {
typedef struct STagFilterResEntry { typedef struct STagFilterResEntry {
SList list; // the linked list of md5 digest, extracted from the serialized tag query condition SList list; // the linked list of md5 digest, extracted from the serialized tag query condition
uint32_t hitTimes; // queried times for current super table uint32_t hitTimes; // queried times for current super table
uint32_t accTime;
} STagFilterResEntry; } STagFilterResEntry;
struct SMetaCache { struct SMetaCache {
...@@ -455,26 +455,37 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv ...@@ -455,26 +455,37 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
return 0; return 0;
} }
static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) {
// ASSERT(keyLen == sizeof(int64_t) * 2);
memcpy(&pBuf[2], key, keyLen);
}
// the format of key:
// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes)
static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) {
buf[0] = (uint64_t) pHashMap;
buf[1] = suid;
setMD5DigestInKey(buf, key, keyLen);
ASSERT(keyLen == sizeof(uint64_t) * 2);
}
int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1,
bool* acquireRes) { bool* acquireRes) {
int32_t vgId = TD_VID(pMeta->pVnode);
// generate the composed key for LRU cache // generate the composed key for LRU cache
SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache; SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache;
SHashObj* pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry; SHashObj* pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry;
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
uint64_t buf[4];
*acquireRes = 0; *acquireRes = 0;
uint64_t key[4];
buf[0] = (uint64_t)pTableMap; initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen);
buf[1] = suid;
memcpy(&buf[2], pKey, keyLen);
taosThreadMutexLock(pLock); taosThreadMutexLock(pLock);
pMeta->pCache->sTagFilterResCache.accTimes += 1; pMeta->pCache->sTagFilterResCache.accTimes += 1;
int32_t len = keyLen + sizeof(uint64_t) * 2; LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN);
LRUHandle* pHandle = taosLRUCacheLookup(pCache, buf, len);
if (pHandle == NULL) { if (pHandle == NULL) {
taosThreadMutexUnlock(pLock); taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -499,7 +510,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK ...@@ -499,7 +510,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK
uint32_t acc = pMeta->pCache->sTagFilterResCache.accTimes; uint32_t acc = pMeta->pCache->sTagFilterResCache.accTimes;
if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) { if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) {
metaInfo("cache hit:%d, total acc:%d, rate:%.2f", (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc); metaInfo("vgId:%d cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc);
} }
taosLRUCacheRelease(pCache, pHandle, false); taosLRUCacheRelease(pCache, pHandle, false);
...@@ -563,10 +574,13 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL ...@@ -563,10 +574,13 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL
// check both the payload size and selectivity ratio // check both the payload size and selectivity ratio
int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen, double selectivityRatio) { int32_t payloadLen, double selectivityRatio) {
int32_t code = 0;
int32_t vgId = TD_VID(pMeta->pVnode);
if (selectivityRatio > tsSelectivityRatio) { if (selectivityRatio > tsSelectivityRatio) {
metaDebug("vgId:%d, suid:%" PRIu64 metaDebug("vgId:%d, suid:%" PRIu64
" failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f", " failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f",
TD_VID(pMeta->pVnode), suid, selectivityRatio, tsSelectivityRatio); vgId, suid, selectivityRatio, tsSelectivityRatio);
taosMemoryFree(pPayload); taosMemoryFree(pPayload);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -574,7 +588,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int ...@@ -574,7 +588,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
if (payloadLen > tsTagFilterResCacheSize) { if (payloadLen > tsTagFilterResCacheSize) {
metaDebug("vgId:%d, suid:%" PRIu64 metaDebug("vgId:%d, suid:%" PRIu64
" failed to add to uid list cache, due to payload length %d greater than threshold %d", " failed to add to uid list cache, due to payload length %d greater than threshold %d",
TD_VID(pMeta->pVnode), suid, payloadLen, tsTagFilterResCacheSize); vgId, suid, payloadLen, tsTagFilterResCacheSize);
taosMemoryFree(pPayload); taosMemoryFree(pPayload);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -583,26 +597,17 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int ...@@ -583,26 +597,17 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
SHashObj* pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry; SHashObj* pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry;
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
// the format of key: uint64_t key[4] = {0};
// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) initCacheKey(key, pTableEntry, suid, pKey, keyLen);
uint64_t buf[4] = {0};
buf[0] = (uint64_t)pTableEntry;
buf[1] = suid;
memcpy(&buf[2], pKey, keyLen);
ASSERT(keyLen == 16);
int32_t code = 0;
taosThreadMutexLock(pLock); taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t)); STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t));
if (pEntry == NULL) { if (pEntry == NULL) {
code = addNewEntry(pTableEntry, pKey, keyLen, suid); code = addNewEntry(pTableEntry, pKey, keyLen, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
} }
} else { } else { // check if it exists or not
// check if it exists or not
size_t size = listNEles(&(*pEntry)->list); size_t size = listNEles(&(*pEntry)->list);
if (size == 0) { if (size == 0) {
tdListAppend(&(*pEntry)->list, pKey); tdListAppend(&(*pEntry)->list, pKey);
...@@ -620,12 +625,11 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int ...@@ -620,12 +625,11 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
} }
// add to cache. // add to cache.
taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) * 2 + keyLen, pPayload, payloadLen, freePayload, NULL, taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freePayload, NULL,
TAOS_LRU_PRIORITY_LOW); TAOS_LRU_PRIORITY_LOW);
_end: _end:
taosThreadMutexUnlock(pLock); taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid,
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid,
(int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry));
return code; return code;
...@@ -633,33 +637,36 @@ _end: ...@@ -633,33 +637,36 @@ _end:
// remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables // remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables
int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) {
int32_t keyLen = sizeof(uint64_t) * 3;
uint64_t p[4] = {0}; uint64_t p[4] = {0};
int32_t vgId = TD_VID(pMeta->pVnode);
SHashObj* pEntryHashMap = pMeta->pCache->sTagFilterResCache.pTableEntry;
p[0] = (uint64_t)pMeta->pCache->sTagFilterResCache.pTableEntry; uint64_t dummy[2] = {0};
p[1] = suid; initCacheKey(p, pEntryHashMap, suid, (char*) &dummy[0], 16);
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
taosThreadMutexLock(pLock); taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t));
STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t));
if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) {
taosThreadMutexUnlock(pLock); taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
(*pEntry)->hitTimes = 0;
SListIter iter = {0}; SListIter iter = {0};
tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD);
SListNode* pNode = NULL; SListNode* pNode = NULL;
while ((pNode = tdListNext(&iter)) != NULL) { while ((pNode = tdListNext(&iter)) != NULL) {
memcpy(&p[2], pNode->data, 16); setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t));
taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, keyLen); taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, TAG_FILTER_RES_KEY_LEN);
} }
(*pEntry)->hitTimes = 0;
tdListEmpty(&(*pEntry)->list); tdListEmpty(&(*pEntry)->list);
taosThreadMutexUnlock(pLock); taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d suid:%"PRId64" cached related tag filter uid list cleared", vgId, suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
source/dnode/vnode/src/tq/tq.c
浏览文件 @ 39eec5e8
此差异已折叠。
source/dnode/vnode/src/tq/tqExec.c
浏览文件 @ 39eec5e8
...@@ -46,11 +46,13 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp ...@@ -46,11 +46,13 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp
static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) { static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) {
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pTq->pVnode->pMeta, 0); metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
// TODO add reference to gurantee success // TODO add reference to gurantee success
if (metaGetTableEntryByUidCache(&mr, uid) < 0) { if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
metaReaderClear(&mr); metaReaderClear(&mr);
return -1; return -1;
} }
for (int32_t i = 0; i < n; i++) { for (int32_t i = 0; i < n; i++) {
char* tbName = taosStrdup(mr.me.name); char* tbName = taosStrdup(mr.me.name);
taosArrayPush(pRsp->blockTbName, &tbName); taosArrayPush(pRsp->blockTbName, &tbName);
...@@ -83,13 +85,16 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs ...@@ -83,13 +85,16 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
while (1) { while (1) {
SSDataBlock* pDataBlock = NULL; SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0; uint64_t ts = 0;
tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId); tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId);
if (qExecTask(task, &pDataBlock, &ts) < 0) { if (qExecTask(task, &pDataBlock, &ts) < 0) {
tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr());
return -1; return -1;
} }
tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock); tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock);
// current scan should be stopped asap, since the rebalance occurs.
if (pDataBlock == NULL) { if (pDataBlock == NULL) {
break; break;
} }
...@@ -99,7 +104,9 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs ...@@ -99,7 +104,9 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
rowCnt += pDataBlock->info.rows; rowCnt += pDataBlock->info.rows;
if (rowCnt >= 4096) break; if (rowCnt >= 4096) {
break;
}
} }
} }
...@@ -113,7 +120,10 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs ...@@ -113,7 +120,10 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
return -1; return -1;
} }
ASSERT(!(pRsp->withTbName || pRsp->withSchema)); if(pRsp->withTbName || pRsp->withSchema){
tqError("get column should not with meta:%d,%d", pRsp->withTbName, pRsp->withSchema);
return -1;
}
return 0; return 0;
} }
......
source/dnode/vnode/src/tq/tqMeta.c
浏览文件 @ 39eec5e8
...@@ -283,7 +283,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { ...@@ -283,7 +283,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
tdbTbcMoveToFirst(pCur); tdbTbcMoveToFirst(pCur);
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
STqHandle handle; STqHandle handle = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
tDecodeSTqHandle(&decoder, &handle); tDecodeSTqHandle(&decoder, &handle);
tDecoderClear(&decoder); tDecoderClear(&decoder);
......
source/dnode/vnode/src/tq/tqPush.c
浏览文件 @ 39eec5e8
...@@ -209,6 +209,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ ...@@ -209,6 +209,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg)); void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg));
int32_t len = msgLen - sizeof(SSubmitReq2Msg); int32_t len = msgLen - sizeof(SSubmitReq2Msg);
int32_t vgId = TD_VID(pTq->pVnode);
if (msgType == TDMT_VND_SUBMIT) { if (msgType == TDMT_VND_SUBMIT) {
// lock push mgr to avoid potential msg lost // lock push mgr to avoid potential msg lost
...@@ -217,7 +218,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) ...@@ -217,7 +218,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr); int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr);
if (numOfRegisteredPush > 0) { if (numOfRegisteredPush > 0) {
tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d", tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d",
pTq->pVnode->config.vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush); vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush);
SArray* cachedKeys = taosArrayInit(0, sizeof(void*)); SArray* cachedKeys = taosArrayInit(0, sizeof(void*));
SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t)); SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t));
...@@ -239,7 +240,10 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) ...@@ -239,7 +240,10 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
pIter = taosHashIterate(pTq->pPushMgr, pIter); pIter = taosHashIterate(pTq->pPushMgr, pIter);
if (pIter == NULL) break; if (pIter == NULL) {
break;
}
STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; STqPushEntry* pPushEntry = *(STqPushEntry**)pIter;
STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey)); STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey));
...@@ -248,31 +252,32 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) ...@@ -248,31 +252,32 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
continue; continue;
} }
if (pPushEntry->dataRsp.reqOffset.version >= ver) { SMqDataRsp* pRsp = pPushEntry->pDataRsp;
tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", if (pRsp->reqOffset.version >= ver) {
pTq->pVnode->config.vgId, pPushEntry->dataRsp.reqOffset.version, ver); tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId,
pRsp->reqOffset.version, ver);
continue; continue;
} }
STqExecHandle* pExec = &pHandle->execHandle; STqExecHandle* pExec = &pHandle->execHandle;
qTaskInfo_t task = pExec->task; qTaskInfo_t task = pExec->task;
SMqDataRsp* pRsp = &pPushEntry->dataRsp;
// prepare scan mem data // prepare scan mem data
SPackedData submit = { SPackedData submit = {
.msgStr = data, .msgStr = data,
.msgLen = len, .msgLen = len,
.ver = ver, .ver = ver,
}; };
qStreamSetScanMemData(task, submit); if(qStreamSetScanMemData(task, submit) != 0){
continue;
}
// here start to scan submit block to extract the subscribed data // here start to scan submit block to extract the subscribed data
while (1) { while (1) {
SSDataBlock* pDataBlock = NULL; SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0; uint64_t ts = 0;
if (qExecTask(task, &pDataBlock, &ts) < 0) { if (qExecTask(task, &pDataBlock, &ts) < 0) {
tqDebug("vgId:%d, tq exec error since %s", pTq->pVnode->config.vgId, terrstr()); tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr());
} }
if (pDataBlock == NULL) { if (pDataBlock == NULL) {
...@@ -283,11 +288,11 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) ...@@ -283,11 +288,11 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
pRsp->blockNum++; pRsp->blockNum++;
} }
tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d", pTq->pVnode->config.vgId, pPushEntry->subKey, tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d", vgId, pPushEntry->subKey, pRsp->blockNum);
pRsp->blockNum);
if (pRsp->blockNum > 0) { if (pRsp->blockNum > 0) {
// set offset // set offset
tqOffsetResetToLog(&pRsp->rspOffset, ver); tqOffsetResetToLog(&pRsp->rspOffset, ver);
// remove from hash // remove from hash
size_t kLen; size_t kLen;
void* key = taosHashGetKey(pIter, &kLen); void* key = taosHashGetKey(pIter, &kLen);
...@@ -309,6 +314,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) ...@@ -309,6 +314,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key); tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key);
} }
} }
taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree); taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree);
taosArrayDestroy(cachedKeyLens); taosArrayDestroy(cachedKeyLens);
taosMemoryFree(data); taosMemoryFree(data);
...@@ -334,9 +340,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) ...@@ -334,9 +340,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
}; };
tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq);
tqProcessSubmitReq(pTq, submit); tqProcessSubmitReq(pTq, submit);
} }
if (msgType == TDMT_VND_DELETE) { if (msgType == TDMT_VND_DELETE) {
tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver);
} }
...@@ -344,3 +350,61 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) ...@@ -344,3 +350,61 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
return 0; return 0;
} }
int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg,
SMqDataRsp* pDataRsp, int32_t type) {
uint64_t consumerId = pRequest->consumerId;
int32_t vgId = TD_VID(pTq->pVnode);
STqHandle* pTqHandle = pHandle;
STqPushEntry* pPushEntry = taosMemoryCalloc(1, sizeof(STqPushEntry));
if (pPushEntry == NULL) {
tqDebug("tmq poll: consumer:0x%" PRIx64 ", vgId:%d failed to malloc, size:%d", consumerId, vgId,
(int32_t)sizeof(STqPushEntry));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pPushEntry->info = pRpcMsg->info;
memcpy(pPushEntry->subKey, pTqHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN);
if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(STaosxRsp));
memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(STaosxRsp));
} else if (type == TMQ_MSG_TYPE__POLL_RSP) {
pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(SMqDataRsp));
memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(SMqDataRsp));
}
SMqRspHead* pHead = &pPushEntry->pDataRsp->head;
pHead->consumerId = consumerId;
pHead->epoch = pRequest->epoch;
pHead->mqMsgType = type;
taosHashPut(pTq->pPushMgr, pTqHandle->subKey, strlen(pTqHandle->subKey), &pPushEntry, sizeof(void*));
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr, total:%d", consumerId,
pTqHandle->subKey, pDataRsp->reqOffset.version, vgId, taosHashGetSize(pTq->pPushMgr));
return 0;
}
int32_t tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) {
int32_t vgId = TD_VID(pTq->pVnode);
STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen);
if (pEntry != NULL) {
uint64_t cId = (*pEntry)->pDataRsp->head.consumerId;
ASSERT(consumerId == cId);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s vgId:%d remove from push mgr, remains:%d", consumerId,
(*pEntry)->subKey, vgId, taosHashGetSize(pTq->pPushMgr) - 1);
if (rspConsumer) { // rsp the old consumer with empty block.
tqPushDataRsp(pTq, *pEntry);
}
taosHashRemove(pTq->pPushMgr, pKey, keyLen);
}
return 0;
}
source/dnode/vnode/src/tq/tqRead.c
浏览文件 @ 39eec5e8
...@@ -183,14 +183,15 @@ end: ...@@ -183,14 +183,15 @@ end:
return tbSuid == realTbSuid; return tbSuid == realTbSuid;
} }
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) { int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
int32_t code = 0; int32_t code = 0;
taosThreadMutexLock(&pHandle->pWalReader->mutex); taosThreadMutexLock(&pHandle->pWalReader->mutex);
int64_t offset = *fetchOffset; int64_t offset = *fetchOffset;
while (1) { while (1) {
if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) { if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return", tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return",
pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset); pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset);
*fetchOffset = offset - 1; *fetchOffset = offset - 1;
code = -1; code = -1;
...@@ -241,6 +242,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea ...@@ -241,6 +242,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea
offset++; offset++;
} }
} }
END: END:
taosThreadMutexUnlock(&pHandle->pWalReader->mutex); taosThreadMutexUnlock(&pHandle->pWalReader->mutex);
return code; return code;
...@@ -294,7 +296,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { ...@@ -294,7 +296,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
// todo set the correct vgId // todo set the correct vgId
tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id); tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id);
if (walReadSeekVer(pReader->pWalReader, ver) < 0) { if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id); tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
return -1; return -1;
} else { } else {
tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id); tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);
...@@ -707,7 +709,10 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD ...@@ -707,7 +709,10 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD
sourceIdx++; sourceIdx++;
targetIdx++; targetIdx++;
} else { } else {
ASSERT(0); for (int32_t i = 0; i < pCol->nVal; i++) {
colDataSetNULL(pColData, i);
}
targetIdx++;
} }
} }
} else { } else {
...@@ -749,7 +754,8 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD ...@@ -749,7 +754,8 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD
sourceIdx++; sourceIdx++;
break; break;
} else { } else {
ASSERT(0); colDataSetNULL(pColData, i);
break;
} }
} }
} }
......
source/dnode/vnode/src/tq/tqSink.c
浏览文件 @ 39eec5e8
...@@ -619,6 +619,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -619,6 +619,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid);
metaReaderClear(&mr); metaReaderClear(&mr);
taosMemoryFree(ctbName); taosMemoryFree(ctbName);
continue;
} }
tbData.uid = mr.me.uid; tbData.uid = mr.me.uid;
......
source/dnode/vnode/src/tsdb/tsdbCache.c
浏览文件 @ 39eec5e8
...@@ -596,7 +596,8 @@ typedef struct { ...@@ -596,7 +596,8 @@ typedef struct {
int64_t lastTs; int64_t lastTs;
} SFSLastNextRowIter; } SFSLastNextRowIter;
static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) { static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter;
int32_t code = 0; int32_t code = 0;
...@@ -632,11 +633,16 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa ...@@ -632,11 +633,16 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
} }
tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
&(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL); &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
state->pMergeTree = &state->mergeTree; state->pMergeTree = &state->mergeTree;
bool hasVal = tMergeTreeNext(&state->mergeTree); bool hasVal = tMergeTreeNext(&state->mergeTree);
if (!hasVal) { if (!hasVal) {
if (tMergeTreeIgnoreEarlierTs(&state->mergeTree)) {
*pIgnoreEarlierTs = true;
*ppRow = NULL;
return code;
}
state->state = SFSLASTNEXTROW_FILESET; state->state = SFSLASTNEXTROW_FILESET;
goto _next_fileset; goto _next_fileset;
} }
...@@ -644,16 +650,13 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa ...@@ -644,16 +650,13 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
} }
case SFSLASTNEXTROW_BLOCKROW: { case SFSLASTNEXTROW_BLOCKROW: {
bool hasVal = false; bool hasVal = false;
do {
state->row = tMergeTreeGetRow(&state->mergeTree); state->row = tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row; *ppRow = &state->row;
hasVal = tMergeTreeNext(&state->mergeTree); hasVal = tMergeTreeNext(&state->mergeTree);
} while (TSDBROW_TS(&state->row) <= state->lastTs && hasVal);
if (TSDBROW_TS(&state->row) <= state->lastTs) { if (TSDBROW_TS(&state->row) <= state->lastTs) {
*pIgnoreEarlierTs = true; *pIgnoreEarlierTs = true;
state->state = SFSLASTNEXTROW_FILESET; *ppRow = NULL;
goto _next_fileset; return code;
} }
*pIgnoreEarlierTs = false; *pIgnoreEarlierTs = false;
...@@ -738,7 +741,8 @@ typedef struct SFSNextRowIter { ...@@ -738,7 +741,8 @@ typedef struct SFSNextRowIter {
int64_t lastTs; int64_t lastTs;
} SFSNextRowIter; } SFSNextRowIter;
static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) { static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SFSNextRowIter *state = (SFSNextRowIter *)iter; SFSNextRowIter *state = (SFSNextRowIter *)iter;
int32_t code = 0; int32_t code = 0;
...@@ -826,8 +830,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie ...@@ -826,8 +830,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
} }
} }
case SFSNEXTROW_BLOCKDATA: case SFSNEXTROW_BLOCKDATA:
_next_datablock:
if (state->iBlock >= 0) { if (state->iBlock >= 0) {
SDataBlk block = {0}; SDataBlk block = {0};
bool skipBlock = true;
int inputColIndex = 0;
tDataBlkReset(&block); tDataBlkReset(&block);
tBlockDataReset(state->pBlockData); tBlockDataReset(state->pBlockData);
...@@ -835,7 +842,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie ...@@ -835,7 +842,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk);
if (block.maxKey.ts <= state->lastTs) { if (block.maxKey.ts <= state->lastTs) {
*pIgnoreEarlierTs = true; *pIgnoreEarlierTs = true;
goto _next_fileset; if (state->pBlockData) {
tBlockDataDestroy(state->pBlockData);
state->pBlockData = NULL;
}
*ppRow = NULL;
return code;
} }
*pIgnoreEarlierTs = false; *pIgnoreEarlierTs = false;
tBlockDataReset(state->pBlockData); tBlockDataReset(state->pBlockData);
...@@ -846,6 +859,42 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie ...@@ -846,6 +859,42 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData); code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData);
if (code) goto _err; if (code) goto _err;
for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
SColData *pColData = &state->pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (inputColIndex < nCols && cid == aCols[inputColIndex++]) {
if (isLast && pColData->numOfValue != 0) {
skipBlock = false;
break;
} else if (pColData->numOfNone != pColData->nVal) {
skipBlock = false;
break;
}
}
}
if (skipBlock) {
if (--state->iBlock < 0) {
tsdbDataFReaderClose(state->pDataFReader);
*state->pDataFReader = NULL;
// resetLastBlockLoadInfo(state->pLoadInfo);
if (state->aBlockIdx) {
// taosArrayDestroy(state->aBlockIdx);
tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
state->aBlockIdxHandle = NULL;
state->aBlockIdx = NULL;
}
state->state = SFSNEXTROW_FILESET;
goto _next_fileset;
} else {
goto _next_datablock;
}
}
state->nRow = state->blockData.nRow; state->nRow = state->blockData.nRow;
state->iRow = state->nRow - 1; state->iRow = state->nRow - 1;
...@@ -952,7 +1001,8 @@ typedef struct SMemNextRowIter { ...@@ -952,7 +1001,8 @@ typedef struct SMemNextRowIter {
// TSDBROW *curRow; // TSDBROW *curRow;
} SMemNextRowIter; } SMemNextRowIter;
static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) { static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SMemNextRowIter *state = (SMemNextRowIter *)iter; SMemNextRowIter *state = (SMemNextRowIter *)iter;
int32_t code = 0; int32_t code = 0;
*pIgnoreEarlierTs = false; *pIgnoreEarlierTs = false;
...@@ -1064,7 +1114,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { ...@@ -1064,7 +1114,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
return deleted; return deleted;
} }
typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs); typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols);
typedef int32_t (*_next_row_clear_fn_t)(void *iter); typedef int32_t (*_next_row_clear_fn_t)(void *iter);
typedef struct { typedef struct {
...@@ -1214,12 +1265,14 @@ _err: ...@@ -1214,12 +1265,14 @@ _err:
} }
// iterate next row non deleted backward ts, version (from high to low) // iterate next row non deleted backward ts, version (from high to low)
static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) { static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast,
int16_t *aCols, int nCols) {
int code = 0; int code = 0;
for (;;) { for (;;) {
for (int i = 0; i < 4; ++i) { for (int i = 0; i < 4; ++i) {
if (pIter->input[i].next && !pIter->input[i].stop) { if (pIter->input[i].next && !pIter->input[i].stop) {
code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs); code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs,
isLast, aCols, nCols);
if (code) goto _err; if (code) goto _err;
if (pIter->input[i].pRow == NULL) { if (pIter->input[i].pRow == NULL) {
...@@ -1350,7 +1403,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo ...@@ -1350,7 +1403,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
do { do {
TSDBROW *pRow = NULL; TSDBROW *pRow = NULL;
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs); nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, NULL, 0);
if (!pRow) { if (!pRow) {
break; break;
...@@ -1480,11 +1533,18 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach ...@@ -1480,11 +1533,18 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
bool ignoreEarlierTs = false; bool ignoreEarlierTs = false;
SArray *pColArray = NULL; SArray *pColArray = NULL;
SColVal *pColVal = &(SColVal){0}; SColVal *pColVal = &(SColVal){0};
int16_t nCols = nLastCol;
int32_t code = initLastColArray(pTSchema, &pColArray); int32_t code = initLastColArray(pTSchema, &pColArray);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t));
if (NULL == aColArray) {
taosArrayDestroy(pColArray);
return TSDB_CODE_OUT_OF_MEMORY;
}
TSKEY lastRowTs = TSKEY_MAX; TSKEY lastRowTs = TSKEY_MAX;
...@@ -1494,7 +1554,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach ...@@ -1494,7 +1554,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
do { do {
TSDBROW *pRow = NULL; TSDBROW *pRow = NULL;
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs); nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray));
if (!pRow) { if (!pRow) {
break; break;
...@@ -1539,11 +1599,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach ...@@ -1539,11 +1599,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
} }
if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { if (!COL_VAL_IS_VALUE(pColVal)) {
taosArrayPush(aColArray, &pColVal->cid);
if (!setNoneCol) {
noneCol = iCol; noneCol = iCol;
setNoneCol = true; setNoneCol = true;
} }
} }
}
if (!setNoneCol) { if (!setNoneCol) {
// done, goto return pColArray // done, goto return pColArray
break; break;
...@@ -1582,6 +1645,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach ...@@ -1582,6 +1645,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
} }
taosArraySet(pColArray, iCol, &lastCol); taosArraySet(pColArray, iCol, &lastCol);
int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ);
taosArrayRemove(aColArray, aColIndex);
} else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { } else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
noneCol = iCol; noneCol = iCol;
setNoneCol = true; setNoneCol = true;
...@@ -1605,6 +1670,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach ...@@ -1605,6 +1670,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
//} //}
nextRowIterClose(&iter); nextRowIterClose(&iter);
taosArrayDestroy(aColArray);
// taosMemoryFreeClear(pTSchema); // taosMemoryFreeClear(pTSchema);
return code; return code;
...@@ -1613,6 +1679,7 @@ _err: ...@@ -1613,6 +1679,7 @@ _err:
// taosMemoryFreeClear(pTSchema); // taosMemoryFreeClear(pTSchema);
*ppLastArray = NULL; *ppLastArray = NULL;
taosArrayDestroy(pColArray); taosArrayDestroy(pColArray);
taosArrayDestroy(aColArray);
return code; return code;
} }
...@@ -1724,6 +1791,15 @@ size_t tsdbCacheGetUsage(SVnode *pVnode) { ...@@ -1724,6 +1791,15 @@ size_t tsdbCacheGetUsage(SVnode *pVnode) {
return usage; return usage;
} }
int32_t tsdbCacheGetElems(SVnode *pVnode) {
int32_t elems = 0;
if (pVnode->pTsdb != NULL) {
elems = taosLRUCacheGetElems(pVnode->pTsdb->lruCache);
}
return elems;
}
static void getBICacheKey(int32_t fid, int64_t commitID, char *key, int *len) { static void getBICacheKey(int32_t fid, int64_t commitID, char *key, int *len) {
struct { struct {
int32_t fid; int32_t fid;
......
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
浏览文件 @ 39eec5e8
...@@ -332,6 +332,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 ...@@ -332,6 +332,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
// retrieve the only one last row of all tables in the uid list. // retrieve the only one last row of all tables in the uid list.
if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) { if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) {
int64_t st = taosGetTimestampUs();
for (int32_t i = 0; i < pr->numOfTables; ++i) { for (int32_t i = 0; i < pr->numOfTables; ++i) {
STableKeyInfo* pKeyInfo = &pr->pTableList[i]; STableKeyInfo* pKeyInfo = &pr->pTableList[i];
...@@ -407,9 +408,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 ...@@ -407,9 +408,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
} }
if (hasNotNullRow) { if (hasNotNullRow) {
double cost = (taosGetTimestampUs() - st) / 1000.0;
if (cost > tsCacheLazyLoadThreshold) {
pr->lastTs = minTs; pr->lastTs = minTs;
} }
} }
}
tsdbCacheRelease(lruCache, h); tsdbCacheRelease(lruCache, h);
} }
...@@ -417,7 +421,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 ...@@ -417,7 +421,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
if (hasRes) { if (hasRes) {
saveOneRow(pLastCols, pResBlock, pr, slotIds, pRes, pr->idstr); saveOneRow(pLastCols, pResBlock, pr, slotIds, pRes, pr->idstr);
} }
} else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) { } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) {
for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) { for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) {
STableKeyInfo* pKeyInfo = &pr->pTableList[i]; STableKeyInfo* pKeyInfo = &pr->pTableList[i];
......
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
浏览文件 @ 39eec5e8
...@@ -29,9 +29,11 @@ struct SLDataIter { ...@@ -29,9 +29,11 @@ struct SLDataIter {
STimeWindow timeWindow; STimeWindow timeWindow;
SVersionRange verRange; SVersionRange verRange;
SSttBlockLoadInfo *pBlockLoadInfo; SSttBlockLoadInfo *pBlockLoadInfo;
bool ignoreEarlierTs;
}; };
SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfSttTrigger) { SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols,
int32_t numOfSttTrigger) {
SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(numOfSttTrigger, sizeof(SSttBlockLoadInfo)); SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(numOfSttTrigger, sizeof(SSttBlockLoadInfo));
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
...@@ -162,7 +164,8 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { ...@@ -162,7 +164,8 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk;
pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1; pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1;
tsdbDebug("last block index list:%d, %d, rowIndex:%d %s", pInfo->blockIndex[0], pInfo->blockIndex[1], pIter->iRow, idStr); tsdbDebug("last block index list:%d, %d, rowIndex:%d %s", pInfo->blockIndex[0], pInfo->blockIndex[1], pIter->iRow,
idStr);
return &pInfo->blockData[pInfo->currentLoadBlockIndex]; return &pInfo->blockData[pInfo->currentLoadBlockIndex];
_exit: _exit:
...@@ -263,7 +266,7 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint ...@@ -263,7 +266,7 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint
int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo,
const char *idStr) { const char *idStr, bool strictTimeRange) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
*pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); *pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
...@@ -340,6 +343,17 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t ...@@ -340,6 +343,17 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
if ((*pIter)->iSttBlk != -1) { if ((*pIter)->iSttBlk != -1) {
(*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk); (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk);
(*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1; (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1;
if ((!backward) && ((strictTimeRange && (*pIter)->pSttBlk->minKey >= (*pIter)->timeWindow.ekey) ||
(!strictTimeRange && (*pIter)->pSttBlk->minKey > (*pIter)->timeWindow.ekey))) {
(*pIter)->pSttBlk = NULL;
}
if (backward && ((strictTimeRange && (*pIter)->pSttBlk->maxKey <= (*pIter)->timeWindow.skey) ||
(!strictTimeRange && (*pIter)->pSttBlk->maxKey < (*pIter)->timeWindow.skey))) {
(*pIter)->pSttBlk = NULL;
(*pIter)->ignoreEarlierTs = true;
}
} }
return code; return code;
...@@ -421,7 +435,7 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { ...@@ -421,7 +435,7 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) {
pBlockData->aUid != NULL) { pBlockData->aUid != NULL) {
i = binarySearchForStartRowIndex((uint64_t *)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward); i = binarySearchForStartRowIndex((uint64_t *)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward);
if (i == -1) { if (i == -1) {
tsdbDebug("failed to find the data in pBlockData, uid:%"PRIu64" , %s", pIter->uid, idStr); tsdbDebug("failed to find the data in pBlockData, uid:%" PRIu64 " , %s", pIter->uid, idStr);
pIter->iRow = -1; pIter->iRow = -1;
return; return;
} }
...@@ -508,7 +522,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { ...@@ -508,7 +522,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
} }
// set start row index // set start row index
pIter->iRow = pIter->backward? pBlockData->nRow-1:0; pIter->iRow = pIter->backward ? pBlockData->nRow - 1 : 0;
} }
} }
...@@ -551,7 +565,7 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR ...@@ -551,7 +565,7 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
bool destroyLoadInfo, const char *idStr) { bool destroyLoadInfo, const char *idStr, bool strictTimeRange) {
pMTree->backward = backward; pMTree->backward = backward;
pMTree->pIter = NULL; pMTree->pIter = NULL;
pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
...@@ -569,11 +583,12 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead ...@@ -569,11 +583,12 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
pMTree->pLoadInfo = pBlockLoadInfo; pMTree->pLoadInfo = pBlockLoadInfo;
pMTree->destroyLoadInfo = destroyLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo;
pMTree->ignoreEarlierTs = false;
for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file
struct SLDataIter *pIter = NULL; struct SLDataIter *pIter = NULL;
code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange,
&pMTree->pLoadInfo[i], pMTree->idStr); &pMTree->pLoadInfo[i], pMTree->idStr, strictTimeRange);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
} }
...@@ -583,6 +598,9 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead ...@@ -583,6 +598,9 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
taosArrayPush(pMTree->pIterList, &pIter); taosArrayPush(pMTree->pIterList, &pIter);
tMergeTreeAddIter(pMTree, pIter); tMergeTreeAddIter(pMTree, pIter);
} else { } else {
if (!pMTree->ignoreEarlierTs) {
pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs;
}
tLDataIterClose(pIter); tLDataIterClose(pIter);
} }
} }
...@@ -596,6 +614,8 @@ _end: ...@@ -596,6 +614,8 @@ _end:
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); }
bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree) { return pMTree->ignoreEarlierTs; }
bool tMergeTreeNext(SMergeTree *pMTree) { bool tMergeTreeNext(SMergeTree *pMTree) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (pMTree->pIter) { if (pMTree->pIter) {
......
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ 39eec5e8
...@@ -315,7 +315,7 @@ static int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) ...@@ -315,7 +315,7 @@ static int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables)
} }
if (pBuf->numOfTables > 0) { if (pBuf->numOfTables > 0) {
STableBlockScanInfo **p = (STableBlockScanInfo**)taosArrayPop(pBuf->pData); STableBlockScanInfo** p = (STableBlockScanInfo**)taosArrayPop(pBuf->pData);
taosMemoryFree(*p); taosMemoryFree(*p);
pBuf->numOfTables /= pBuf->numPerBucket; pBuf->numOfTables /= pBuf->numPerBucket;
} }
...@@ -919,7 +919,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN ...@@ -919,7 +919,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
pBlockNum->numOfBlocks += 1; pBlockNum->numOfBlocks += 1;
} }
if ((pScanInfo->pBlockList != NULL )&& (taosArrayGetSize(pScanInfo->pBlockList) > 0)) { if ((pScanInfo->pBlockList != NULL) && (taosArrayGetSize(pScanInfo->pBlockList) > 0)) {
numOfQTable += 1; numOfQTable += 1;
} }
} }
...@@ -2481,7 +2481,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan ...@@ -2481,7 +2481,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
pScanInfo->uid, pReader->idStr); pScanInfo->uid, pReader->idStr);
int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC),
pReader->pFileReader, pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pReader->pFileReader, pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange,
pLBlockReader->pInfo, false, pReader->idStr); pLBlockReader->pInfo, false, pReader->idStr, false);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return false; return false;
} }
...@@ -3927,7 +3927,8 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n ...@@ -3927,7 +3927,8 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n
if (code) { if (code) {
return code; return code;
} }
pReader->status.uidList.tableUidList = (uint64_t*)taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); pReader->status.uidList.tableUidList =
(uint64_t*)taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num);
} }
taosHashClear(pReader->status.pTableMap); taosHashClear(pReader->status.pTableMap);
......
source/dnode/vnode/src/vnd/vnodeQuery.c
浏览文件 @ 39eec5e8
...@@ -382,6 +382,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { ...@@ -382,6 +382,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
pLoad->syncRestore = state.restored; pLoad->syncRestore = state.restored;
pLoad->syncCanRead = state.canRead; pLoad->syncCanRead = state.canRead;
pLoad->cacheUsage = tsdbCacheGetUsage(pVnode); pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
pLoad->numOfCachedTables = tsdbCacheGetElems(pVnode);
pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
pLoad->totalStorage = (int64_t)3 * 1073741824; pLoad->totalStorage = (int64_t)3 * 1073741824;
......
source/dnode/vnode/src/vnd/vnodeSvr.c
浏览文件 @ 39eec5e8
...@@ -306,13 +306,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp ...@@ -306,13 +306,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
void *pReq; void *pReq;
int32_t len; int32_t len;
int32_t ret; int32_t ret;
/*
if (!pVnode->inUse) {
terrno = TSDB_CODE_VND_NO_AVAIL_BUFPOOL;
vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr());
return -1;
}
*/
if (version <= pVnode->state.applied) { if (version <= pVnode->state.applied) {
vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version,
pVnode->state.applied); pVnode->state.applied);
...@@ -326,8 +320,8 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp ...@@ -326,8 +320,8 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm);
ASSERT(pVnode->state.applied + 1 == version); ASSERT(pVnode->state.applied + 1 == version);
pVnode->state.applied = version; atomic_store_64(&pVnode->state.applied, version);
pVnode->state.applyTerm = pMsg->info.conn.applyTerm; atomic_store_64(&pVnode->state.applyTerm, pMsg->info.conn.applyTerm);
if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; if (!syncUtilUserCommit(pMsg->msgType)) goto _exit;
......
source/dnode/vnode/src/vnd/vnodeSync.c
浏览文件 @ 39eec5e8
...@@ -433,7 +433,23 @@ static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsm ...@@ -433,7 +433,23 @@ static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsm
} }
static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
if (pMsg->code == 0) {
return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); return vnodeSyncApplyMsg(pFsm, pMsg, pMeta);
}
const STraceId *trace = &pMsg->info.traceId;
SVnode *pVnode = pFsm->data;
vnodePostBlockMsg(pVnode, pMsg);
SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info};
if (rsp.info.handle != NULL) {
tmsgSendRsp(&rsp);
}
vGTrace("vgId:%d, msg:%p is freed, code:0x%x index:%" PRId64, TD_VID(pVnode), pMsg, rsp.code, pMeta->index);
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
return 0;
} }
static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
...@@ -443,6 +459,11 @@ static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const ...@@ -443,6 +459,11 @@ static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const
return 0; return 0;
} }
static SyncIndex vnodeSyncAppliedIndex(const SSyncFSM *pFSM) {
SVnode *pVnode = pFSM->data;
return atomic_load_64(&pVnode->state.applied);
}
static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s",
...@@ -505,21 +526,26 @@ static int32_t vnodeSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *p ...@@ -505,21 +526,26 @@ static int32_t vnodeSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *p
return code; return code;
} }
static void vnodeRestoreFinish(const SSyncFSM *pFsm) { static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
SyncIndex appliedIdx = -1;
do { do {
int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); appliedIdx = vnodeSyncAppliedIndex(pFsm);
if (itemSize == 0) { ASSERT(appliedIdx <= commitIdx);
vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId); if (appliedIdx == commitIdx) {
vInfo("vgId:%d, no items to be applied, restore finish", pVnode->config.vgId);
break; break;
} else { } else {
vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId, itemSize); vInfo("vgId:%d, restore not finish since %" PRId64 " items to be applied. commit-index:%" PRId64
", applied-index:%" PRId64,
pVnode->config.vgId, commitIdx - appliedIdx, commitIdx, appliedIdx);
taosMsleep(10); taosMsleep(10);
} }
} while (true); } while (true);
walApplyVer(pVnode->pWal, pVnode->state.applied); ASSERT(commitIdx == vnodeSyncAppliedIndex(pFsm));
walApplyVer(pVnode->pWal, commitIdx);
pVnode->restored = true; pVnode->restored = true;
vInfo("vgId:%d, sync restore finished", pVnode->config.vgId); vInfo("vgId:%d, sync restore finished", pVnode->config.vgId);
...@@ -569,6 +595,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { ...@@ -569,6 +595,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
pFsm->data = pVnode; pFsm->data = pVnode;
pFsm->FpCommitCb = vnodeSyncCommitMsg; pFsm->FpCommitCb = vnodeSyncCommitMsg;
pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex;
pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg; pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg;
pFsm->FpRollBackCb = vnodeSyncRollBackMsg; pFsm->FpRollBackCb = vnodeSyncRollBackMsg;
pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshotInfo; pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshotInfo;
......
source/libs/command/src/command.c
浏览文件 @ 39eec5e8
...@@ -307,7 +307,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch ...@@ -307,7 +307,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
bool existLeaderRole(TAOS_ROW row, TAOS_FIELD* fields, int nFields) { bool existLeaderRole(TAOS_ROW row, TAOS_FIELD* fields, int nFields) {
// vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | // vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode |
// v4_status | cacheload | tsma | // v4_status | cacheload | tsma |
if (nFields != 13) { if (nFields != 14) {
return false; return false;
} }
......
source/libs/executor/src/executor.c
浏览文件 @ 39eec5e8
...@@ -221,12 +221,12 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, ...@@ -221,12 +221,12 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) { qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) {
if (msg == NULL) { if (msg == NULL) {
// create raw scan // create raw scan
SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
if (NULL == pTaskInfo) { if (NULL == pTaskInfo) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTaskInfo->cost.created = taosGetTimestampUs(); pTaskInfo->cost.created = taosGetTimestampUs();
...@@ -715,7 +715,6 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) { ...@@ -715,7 +715,6 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) {
int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
if (pTaskInfo == NULL) { if (pTaskInfo == NULL) {
return TSDB_CODE_QRY_INVALID_QHANDLE; return TSDB_CODE_QRY_INVALID_QHANDLE;
} }
...@@ -723,7 +722,6 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { ...@@ -723,7 +722,6 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) {
qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); qDebug("%s execTask async killed", GET_TASKID(pTaskInfo));
setTaskKilled(pTaskInfo, rspCode); setTaskKilled(pTaskInfo, rspCode);
qStopTaskOperators(pTaskInfo); qStopTaskOperators(pTaskInfo);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -992,19 +990,16 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) { ...@@ -992,19 +990,16 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) {
SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
return &pTaskInfo->streamInfo.metaRsp; return &pTaskInfo->streamInfo.metaRsp;
} }
int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo) { int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
return pTaskInfo->streamInfo.prepareStatus.uid; return pTaskInfo->streamInfo.prepareStatus.uid;
} }
int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal));
return 0; return 0;
} }
...@@ -1040,20 +1035,12 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s ...@@ -1040,20 +1035,12 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
#if 0
int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t scanVer) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
ASSERT(pTaskInfo->streamInfo.pReq == NULL);
pTaskInfo->streamInfo.pReq = pReq;
pTaskInfo->streamInfo.scanVer = scanVer;
return 0;
}
#endif
int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
ASSERT((pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE )&& (pTaskInfo->streamInfo.submit.msgStr == NULL)); if((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)){
qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr);
return -1;
}
qDebug("set the submit block for future scan"); qDebug("set the submit block for future scan");
pTaskInfo->streamInfo.submit = submit; pTaskInfo->streamInfo.submit = submit;
...@@ -1063,7 +1050,6 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { ...@@ -1063,7 +1050,6 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SOperatorInfo* pOperator = pTaskInfo->pRoot; SOperatorInfo* pOperator = pTaskInfo->pRoot;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
pTaskInfo->streamInfo.prepareStatus = *pOffset; pTaskInfo->streamInfo.prepareStatus = *pOffset;
pTaskInfo->streamInfo.returned = 0; pTaskInfo->streamInfo.returned = 0;
...@@ -1076,7 +1062,10 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1076,7 +1062,10 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
// TODO add more check // TODO add more check
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
ASSERT(pOperator->numOfDownstream == 1); if(pOperator->numOfDownstream != 1){
qError("pOperator->numOfDownstream != 1:%d", pOperator->numOfDownstream);
return -1;
}
pOperator = pOperator->pDownstream[0]; pOperator = pOperator->pDownstream[0];
} }
...@@ -1087,6 +1076,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1087,6 +1076,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pTSInfo->base.dataReader = NULL; pTSInfo->base.dataReader = NULL;
// let's seek to the next version in wal file // let's seek to the next version in wal file
if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) { if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) {
qError("tqSeekVer failed ver:%" PRId64, pOffset->version + 1);
return -1; return -1;
} }
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
...@@ -1101,6 +1091,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1101,6 +1091,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
uid = pTableInfo->uid; uid = pTableInfo->uid;
ts = INT64_MIN; ts = INT64_MIN;
} else { } else {
qError("uid == 0 and tablelist size is 0");
return -1; return -1;
} }
} }
...@@ -1124,7 +1115,10 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1124,7 +1115,10 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
} }
// TODO after dropping table, table may not found // TODO after dropping table, table may not found
ASSERT(found); if(!found){
qError("uid not found in tablelist %" PRId64, uid);
return -1;
}
if (pTableScanInfo->base.dataReader == NULL) { if (pTableScanInfo->base.dataReader == NULL) {
STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
...@@ -1133,7 +1127,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1133,7 +1127,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num,
pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 || pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 ||
pTableScanInfo->base.dataReader == NULL) { pTableScanInfo->base.dataReader == NULL) {
ASSERT(0); qError("tsdbReaderOpen failed. uid:%" PRIi64, pOffset->uid);
return -1;
} }
} }
...@@ -1148,7 +1143,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1148,7 +1143,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid,
ts, pTableScanInfo->currentTable, numOfTables); ts, pTableScanInfo->currentTable, numOfTables);
} else { } else {
ASSERT(0); qError("invalid pOffset->type:%d", pOffset->type);
return -1;
} }
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
SStreamRawScanInfo* pInfo = pOperator->info; SStreamRawScanInfo* pInfo = pOperator->info;
...@@ -1180,7 +1176,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1180,7 +1176,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); int32_t size = tableListGetSize(pTaskInfo->pTableInfoList);
ASSERT(size == 1);
tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL);
...@@ -1209,7 +1204,10 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1209,7 +1204,10 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle;
assert(pMsg->info.ahandle != NULL); if(pMsg->info.ahandle == NULL){
qError("pMsg->info.ahandle is NULL");
return;
}
SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; SDataBuf buf = {.len = pMsg->contLen, .pData = NULL};
......
source/libs/executor/src/groupoperator.c
浏览文件 @ 39eec5e8
...@@ -999,14 +999,14 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* ...@@ -999,14 +999,14 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
memset(tbName, 0, TSDB_TABLE_NAME_LEN); memset(tbName, 0, TSDB_TABLE_NAME_LEN);
int32_t len = 0; int32_t len = 0;
if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) { if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) {
len = TMIN(sizeof(TSDB_DATA_NULL_STR), TSDB_TABLE_NAME_LEN - 1); len = 1;
memcpy(tbName, TSDB_DATA_NULL_STR, len); tbName[0] = 0;
} else { } else {
void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1);
len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, varDataVal(pData), len); memcpy(tbName, varDataVal(pData), len);
}
streamStatePutParName(pState, groupId, tbName); streamStatePutParName(pState, groupId, tbName);
}
memcpy(pTmpBlock->info.parTbName, tbName, len); memcpy(pTmpBlock->info.parTbName, tbName, len);
pDestBlock->info.rows--; pDestBlock->info.rows--;
} else { } else {
......
source/libs/executor/src/timewindowoperator.c
浏览文件 @ 39eec5e8
...@@ -939,7 +939,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul ...@@ -939,7 +939,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
TSKEY ekey = ascScan ? win.ekey : win.skey; TSKEY ekey = ascScan ? win.ekey : win.skey;
int32_t forwardRows = int32_t forwardRows =
getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
ASSERT(forwardRows > 0);
// prev time window not interpolation yet. // prev time window not interpolation yet.
if (pInfo->timeWindowInterpo) { if (pInfo->timeWindowInterpo) {
...@@ -3912,7 +3911,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl ...@@ -3912,7 +3911,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
blockDataEnsureCapacity(pAggSup->pScanBlock, rows); blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
for (int32_t i = 0; i < rows; i += winRows) { for (int32_t i = 0; i < rows; i += winRows) {
if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) {
i++; i++;
continue; continue;
} }
......
source/libs/function/src/udfd.c
浏览文件 @ 39eec5e8
此差异已折叠。
source/libs/stream/src/streamExec.c
浏览文件 @ 39eec5e8
此差异已折叠。
source/libs/sync/src/syncMain.c
浏览文件 @ 39eec5e8
此差异已折叠。
source/libs/sync/src/syncPipeline.c
浏览文件 @ 39eec5e8
此差异已折叠。
source/util/src/terror.c
浏览文件 @ 39eec5e8
此差异已折叠。
source/util/src/tlrucache.c
浏览文件 @ 39eec5e8
此差异已折叠。
tests/parallel_test/cases.task
浏览文件 @ 39eec5e8
此差异已折叠。
tests/requirements.txt
浏览文件 @ 39eec5e8
此差异已折叠。
tests/script/tsim/query/tableCount.sim 0 → 100644
浏览文件 @ 39eec5e8
此差异已折叠。
tests/script/tsim/stream/state1.sim 0 → 100644
浏览文件 @ 39eec5e8
此差异已折叠。
tests/system-test/2-query/odbc.py
浏览文件 @ 39eec5e8
此差异已折叠。
utils/test/c/tmqSim.c
浏览文件 @ 39eec5e8
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或