Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1193
Star 22018
Fork 4786
T
TDengine
“699db3f1181dae469aadbf0c0871969774ba5452”上不存在“mobile/test/fpga/test_ssd.cpp”
提交 6811a6b2 编写于 作者: P plum-lihui
浏览文件
操作

Merge branch 'main' into lihui3.0/test

上级 ecd39d33 b1b56b16
隐藏空白更改
内联 并排
Showing with 2065 addition and 574 deletion
cmake/cmake.version
浏览文件 @ 6811a6b2
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER) IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER}) SET(TD_VER_NUMBER ${VERNUMBER})
ELSE () ELSE ()
SET(TD_VER_NUMBER "3.0.3.0") SET(TD_VER_NUMBER "3.0.3.1")
ENDIF () ENDIF ()
IF (DEFINED VERCOMPATIBLE) IF (DEFINED VERCOMPATIBLE)
......
include/libs/executor/executor.h
浏览文件 @ 6811a6b2
...@@ -67,9 +67,10 @@ typedef enum { ...@@ -67,9 +67,10 @@ typedef enum {
* Create the exec task for stream mode * Create the exec task for stream mode
* @param pMsg * @param pMsg
* @param SReadHandle * @param SReadHandle
* @param vgId
* @return * @return
*/ */
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId);
/** /**
* Create the exec task for queue mode * Create the exec task for queue mode
...@@ -77,7 +78,15 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); ...@@ -77,7 +78,15 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers);
* @param SReadHandle * @param SReadHandle
* @return * @return
*/ */
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema); qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, SSchemaWrapper** pSchema);
/**
* set the task Id, usually used by message queue process
* @param tinfo
* @param taskId
* @param queryId
*/
void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
/** /**
......
include/libs/monitor/monitor.h
浏览文件 @ 6811a6b2
...@@ -147,9 +147,9 @@ typedef struct { ...@@ -147,9 +147,9 @@ typedef struct {
} SMonStbInfo; } SMonStbInfo;
typedef struct { typedef struct {
int32_t expire_time; uint32_t expire_time;
int64_t timeseries_used; int64_t timeseries_used;
int64_t timeseries_total; int64_t timeseries_total;
} SMonGrantInfo; } SMonGrantInfo;
typedef struct { typedef struct {
......
include/util/tdef.h
浏览文件 @ 6811a6b2
...@@ -248,7 +248,7 @@ typedef enum ELogicConditionType { ...@@ -248,7 +248,7 @@ typedef enum ELogicConditionType {
#define TSDB_AUTH_LEN 16 #define TSDB_AUTH_LEN 16
#define TSDB_PASSWORD_LEN 32 #define TSDB_PASSWORD_LEN 32
#define TSDB_USET_PASSWORD_LEN 129 #define TSDB_USET_PASSWORD_LEN 129
#define TSDB_VERSION_LEN 12 #define TSDB_VERSION_LEN 32
#define TSDB_LABEL_LEN 8 #define TSDB_LABEL_LEN 8
#define TSDB_JOB_STATUS_LEN 32 #define TSDB_JOB_STATUS_LEN 32
......
packaging/testpackage.sh
浏览文件 @ 6811a6b2
...@@ -107,7 +107,7 @@ elif [ ${testFile} = "tools" ];then ...@@ -107,7 +107,7 @@ elif [ ${testFile} = "tools" ];then
originTdpPath="taosTools-${originversion}" originTdpPath="taosTools-${originversion}"
packageName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" packageName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}"
originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}" originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}"
installCmd="install-tools.sh" installCmd="install-taostools.sh"
fi fi
...@@ -333,7 +333,7 @@ if [[ ${packageName} =~ "Lite" ]] || ([[ ${packageName} =~ "x64" ]] && [[ ${p ...@@ -333,7 +333,7 @@ if [[ ${packageName} =~ "Lite" ]] || ([[ ${packageName} =~ "x64" ]] && [[ ${p
wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web
tar xf taosTools-2.1.3-Linux-x64.tar.gz tar xf taosTools-2.1.3-Linux-x64.tar.gz
fi fi
cd taosTools-2.1.3 && bash install-tools.sh cd taosTools-2.1.3 && bash install-taostools.sh
elif ([[ ${packageName} =~ "arm64" ]] && [[ ${packageName} =~ "client" ]]);then elif ([[ ${packageName} =~ "arm64" ]] && [[ ${packageName} =~ "client" ]]);then
echoColor G "===== install taos-tools arm when package is arm64-client =====" echoColor G "===== install taos-tools arm when package is arm64-client ====="
cd ${installPath} cd ${installPath}
...@@ -342,7 +342,7 @@ elif ([[ ${packageName} =~ "arm64" ]] && [[ ${packageName} =~ "client" ]]);then ...@@ -342,7 +342,7 @@ elif ([[ ${packageName} =~ "arm64" ]] && [[ ${packageName} =~ "client" ]]);then
tar xf taosTools-2.1.3-Linux-arm64.tar.gz tar xf taosTools-2.1.3-Linux-arm64.tar.gz
fi fi
cd taosTools-2.1.3 && bash install-tools.sh cd taosTools-2.1.3 && bash install-taostools.sh
fi fi
echoColor G "===== start TDengine =====" echoColor G "===== start TDengine ====="
...@@ -361,18 +361,18 @@ rm -rf ${installPath}/${tdPath}/ ...@@ -361,18 +361,18 @@ rm -rf ${installPath}/${tdPath}/
# cd ${installPath} # cd ${installPath}
# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . # wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
# tar xf taosTools-2.1.2-Linux-x64.tar.gz # tar xf taosTools-2.1.2-Linux-x64.tar.gz
# cd taosTools-2.1.2 && bash install-tools.sh # cd taosTools-2.1.2 && bash install-taostools.sh
# elif [[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "deb" ]] ;then # elif [[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "deb" ]] ;then
# echoColor G "===== install taos-tools when package is lite or client =====" # echoColor G "===== install taos-tools when package is lite or client ====="
# cd ${installPath} # cd ${installPath}
# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . # wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
# tar xf taosTools-2.1.2-Linux-x64.tar.gz # tar xf taosTools-2.1.2-Linux-x64.tar.gz
# cd taosTools-2.1.2 && bash install-tools.sh # cd taosTools-2.1.2 && bash install-taostools.sh
# elif [[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "rpm" ]] ;then # elif [[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "rpm" ]] ;then
# echoColor G "===== install taos-tools when package is lite or client =====" # echoColor G "===== install taos-tools when package is lite or client ====="
# cd ${installPath} # cd ${installPath}
# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . # wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
# tar xf taosTools-2.1.2-Linux-x64.tar.gz # tar xf taosTools-2.1.2-Linux-x64.tar.gz
# cd taosTools-2.1.2 && bash install-tools.sh # cd taosTools-2.1.2 && bash install-taostools.sh
# fi # fi
packaging/tools/com.taosdata.taoskeeper.plist 0 → 100644
浏览文件 @ 6811a6b2
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.tdengine.taoskeeper</string>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/taoskeeper</string>
</array>
<key>ProcessType</key>
<string>Interactive</string>
<key>Disabled</key>
<false/>
<key>RunAtLoad</key>
<false/>
<key>LaunchOnlyOnce</key>
<false/>
<key>SessionCreate</key>
<true/>
<key>ExitTimeOut</key>
<integer>600</integer>
<key>KeepAlive</key>
<dict>
<key>SuccessfulExit</key>
<false/>
<key>AfterInitialDemand</key>
<true/>
</dict>
<key>Program</key>
<string>/usr/local/bin/taoskeeper</string>
</dict>
</plist>
\ No newline at end of file
packaging/tools/makepkg.sh
浏览文件 @ 6811a6b2
...@@ -53,7 +53,7 @@ if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then ...@@ -53,7 +53,7 @@ if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
cd ${top_dir}/tools/taos-tools/packaging/deb cd ${top_dir}/tools/taos-tools/packaging/deb
[ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
taostools_ver=$(git tag |grep -v taos | sort | tail -1) taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1)
taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}" taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}"
cd ${curr_dir} cd ${curr_dir}
......
packaging/tools/post.sh
浏览文件 @ 6811a6b2
...@@ -582,6 +582,11 @@ function install_service_on_launchctl() { ...@@ -582,6 +582,11 @@ function install_service_on_launchctl() {
${csudo}cp ${install_main_dir}/service/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist || : ${csudo}cp ${install_main_dir}/service/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist || :
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist || : ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist || :
fi fi
if [ -f ${install_main_dir}/service/com.taosdata.taoskeeper.plist ]; then
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist > /dev/null 2>&1 || :
${csudo}cp ${install_main_dir}/service/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || :
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || :
fi
} }
function install_taosadapter_service() { function install_taosadapter_service() {
......
source/client/inc/clientSml.h
浏览文件 @ 6811a6b2
...@@ -70,7 +70,7 @@ extern "C" { ...@@ -70,7 +70,7 @@ extern "C" {
#define VALUE_LEN 6 #define VALUE_LEN 6
#define OTD_JSON_FIELDS_NUM 4 #define OTD_JSON_FIELDS_NUM 4
#define MAX_RETRY_TIMES 5 #define MAX_RETRY_TIMES 100
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
typedef enum { typedef enum {
......
source/client/src/clientEnv.c
浏览文件 @ 6811a6b2
...@@ -543,7 +543,7 @@ void taos_init_imp(void) { ...@@ -543,7 +543,7 @@ void taos_init_imp(void) {
if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
// ignore create log failed, only print // ignore create log failed, only print
printf(" WARING: Create taoslog failed. configDir=%s\n", configDir); printf(" WARING: Create taoslog failed:%s. configDir=%s\n", strerror(errno), configDir);
} }
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
......
source/client/src/clientImpl.c
浏览文件 @ 6811a6b2
...@@ -192,7 +192,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, ...@@ -192,7 +192,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
(*pRequest)->sqlLen = sqlLen; (*pRequest)->sqlLen = sqlLen;
(*pRequest)->validateOnly = validateSql; (*pRequest)->validateOnly = validateSql;
SSyncQueryParam* newpParam; SSyncQueryParam* newpParam = NULL;
if (param == NULL) { if (param == NULL) {
newpParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); newpParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (newpParam == NULL) { if (newpParam == NULL) {
......
source/client/src/clientMain.c
浏览文件 @ 6811a6b2
...@@ -271,8 +271,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { ...@@ -271,8 +271,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
SReqResultInfo *pResultInfo; SReqResultInfo *pResultInfo;
if (msg->resIter == -1) { if (msg->resIter == -1) {
pResultInfo = tmqGetNextResInfo(res, true); pResultInfo = tmqGetNextResInfo(res, true);
tscDebug("consumer:0x%" PRIx64 ", vgId:%d, numOfRows:%" PRId64 ", total rows:%" PRId64, msg->rsp.head.consumerId,
msg->vgId, pResultInfo->numOfRows, pResultInfo->totalRows);
} else { } else {
pResultInfo = tmqGetCurResInfo(res); pResultInfo = tmqGetCurResInfo(res);
} }
...@@ -287,9 +285,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { ...@@ -287,9 +285,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL; return NULL;
} }
tscDebug("consumer:0x%" PRIx64 " vgId:%d, numOfRows:%" PRId64 ", total rows:%" PRId64, msg->rsp.head.consumerId,
msg->vgId, pResultInfo->numOfRows, pResultInfo->totalRows);
doSetOneRowPtr(pResultInfo); doSetOneRowPtr(pResultInfo);
pResultInfo->current += 1; pResultInfo->current += 1;
return pResultInfo->row; return pResultInfo->row;
......
source/client/src/clientMsgHandler.c
浏览文件 @ 6811a6b2
...@@ -506,6 +506,9 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { ...@@ -506,6 +506,9 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true); code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
} }
if(code != 0){
taosMemoryFree(pRes);
}
tFreeSShowVariablesRsp(&rsp); tFreeSShowVariablesRsp(&rsp);
} }
......
source/client/src/clientRawBlockWrite.c
浏览文件 @ 6811a6b2
...@@ -1373,6 +1373,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch ...@@ -1373,6 +1373,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname);
goto end; goto end;
} }
// uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
pQuery = smlInitHandle(); pQuery = smlInitHandle();
if (pQuery == NULL) { if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
...@@ -1380,6 +1382,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch ...@@ -1380,6 +1382,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData));
// uError("td23101 1vgId:%d, numEps:%d, name:%s, uid:%"PRIu64, vgData.vgId, vgData.epSet.numOfEps, tbname, pTableMeta->uid);
code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false); code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
...@@ -1723,10 +1726,15 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) ...@@ -1723,10 +1726,15 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
tDecoderClear(&decoderTmp); tDecoderClear(&decoderTmp);
} }
if (pCreateReqDst) { SVgroupInfo vg;
code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg);
if (code != TSDB_CODE_SUCCESS) {
uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName);
goto end;
}
if (pCreateReqDst) { // change stable name to get meta
strcpy(pName.tname, pCreateReqDst->ctb.stbName); strcpy(pName.tname, pCreateReqDst->ctb.stbName);
} else {
strcpy(pName.tname, tbName);
} }
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
...@@ -1739,13 +1747,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) ...@@ -1739,13 +1747,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
goto end; goto end;
} }
SVgroupInfo vg;
code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg);
if (code != TSDB_CODE_SUCCESS) {
uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName);
goto end;
}
if (pCreateReqDst) { if (pCreateReqDst) {
pTableMeta->vgId = vg.vgId; pTableMeta->vgId = vg.vgId;
pTableMeta->uid = pCreateReqDst->uid; pTableMeta->uid = pCreateReqDst->uid;
......
source/client/src/clientSml.c
浏览文件 @ 6811a6b2
...@@ -117,7 +117,7 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u ...@@ -117,7 +117,7 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u
if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) { if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) {
int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS];
if (unit > INT64_MAX / tsInt64) { if (tsInt64 != 0 && unit > INT64_MAX / tsInt64) {
return -1; return -1;
} }
tsInt64 *= unit; tsInt64 *= unit;
...@@ -637,7 +637,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO ...@@ -637,7 +637,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
for (int j = 0; j < taosArrayGetSize(cols); ++j) { for (int j = 0; j < taosArrayGetSize(cols); ++j) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j);
ESchemaAction action = SCHEMA_ACTION_NULL; ESchemaAction action = SCHEMA_ACTION_NULL;
smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
if(code != 0){
return code;
}
if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) {
SField field = {0}; SField field = {0};
field.type = kv->type; field.type = kv->type;
...@@ -646,6 +649,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO ...@@ -646,6 +649,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
taosArrayPush(results, &field); taosArrayPush(results, &field);
} else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen);
if(index == NULL){
uError("smlBuildFieldsList get error, key:%s", kv->key);
return TSDB_CODE_SML_INVALID_DATA;
}
uint16_t newIndex = *index; uint16_t newIndex = *index;
if (isTag) newIndex -= numOfCols; if (isTag) newIndex -= numOfCols;
SField *field = (SField *)taosArrayGet(results, newIndex); SField *field = (SField *)taosArrayGet(results, newIndex);
...@@ -743,6 +750,7 @@ end: ...@@ -743,6 +750,7 @@ end:
} }
static int32_t smlModifyDBSchemas(SSmlHandle *info) { static int32_t smlModifyDBSchemas(SSmlHandle *info) {
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema);
if (info->dataFormat && !info->needModifySchema) { if (info->dataFormat && !info->needModifySchema) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -772,11 +780,19 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { ...@@ -772,11 +780,19 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) {
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname);
SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true);
smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname);
goto end;
}
code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname);
goto end;
}
code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
...@@ -804,6 +820,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { ...@@ -804,6 +820,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end; goto end;
} }
if (action != SCHEMA_ACTION_NULL) { if (action != SCHEMA_ACTION_NULL) {
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, action);
SArray *pColumns = SArray *pColumns =
taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray *pTags = SArray *pTags =
...@@ -820,8 +837,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { ...@@ -820,8 +837,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
taosArrayPush(pTags, &field); taosArrayPush(pTags, &field);
} }
} }
smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags,
pTableMeta->tableInfo.numOfColumns, true); pTableMeta->tableInfo.numOfColumns, true);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname);
goto end;
}
code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
...@@ -851,6 +872,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { ...@@ -851,6 +872,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end; goto end;
} }
if (action != SCHEMA_ACTION_NULL) { if (action != SCHEMA_ACTION_NULL) {
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, action);
SArray *pColumns = SArray *pColumns =
taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray *pTags = SArray *pTags =
...@@ -868,8 +890,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { ...@@ -868,8 +890,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
} }
smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns,
pTableMeta->tableInfo.numOfColumns, false); pTableMeta->tableInfo.numOfColumns, false);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname);
goto end;
}
code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
...@@ -913,15 +939,19 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { ...@@ -913,15 +939,19 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
sTableData->tableMeta = pTableMeta; sTableData->tableMeta = pTableMeta;
uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion)
tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
} }
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema);
return 0; return 0;
end: end:
taosHashCleanup(hashTmp); taosHashCleanup(hashTmp);
taosMemoryFreeClear(pTableMeta); taosMemoryFreeClear(pTableMeta);
// catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema);
return code; return code;
} }
...@@ -997,8 +1027,9 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols ...@@ -997,8 +1027,9 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols
} else { } else {
size_t tmp = taosArrayGetSize(metaArray); size_t tmp = taosArrayGetSize(metaArray);
if (tmp > INT16_MAX) { if (tmp > INT16_MAX) {
smlBuildInvalidDataMsg(msg, "too many cols or tags", kv->key);
uError("too many cols or tags"); uError("too many cols or tags");
return -1; return TSDB_CODE_SML_INVALID_DATA;
} }
int16_t size = tmp; int16_t size = tmp;
int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES);
...@@ -1097,6 +1128,9 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { ...@@ -1097,6 +1128,9 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
} }
if (taos != NULL) { if (taos != NULL) {
info->taos = acquireTscObj(*(int64_t *)taos); info->taos = acquireTscObj(*(int64_t *)taos);
if(info->taos == NULL){
goto cleanup;
}
code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code);
...@@ -1145,19 +1179,23 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { ...@@ -1145,19 +1179,23 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
} }
static int32_t smlParseLineBottom(SSmlHandle *info) { static int32_t smlParseLineBottom(SSmlHandle *info) {
uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
if (info->dataFormat) return TSDB_CODE_SUCCESS; if (info->dataFormat) return TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < info->lineNum; i++) { for (int32_t i = 0; i < info->lineNum; i++) {
SSmlLineInfo *elements = info->lines + i; SSmlLineInfo *elements = info->lines + i;
SSmlTableInfo *tinfo = NULL; SSmlTableInfo *tinfo = NULL;
if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen); SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen);
if(tmp) tinfo = *tmp;
} else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
elements->measureLen + elements->tagsLen); elements->measureLen + elements->tagsLen);
if(tmp) tinfo = *tmp;
} else { } else {
tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
elements->measureLen + elements->tagsLen); elements->measureLen + elements->tagsLen);
if(tmp) tinfo = *tmp;
} }
if (tinfo == NULL) { if (tinfo == NULL) {
...@@ -1184,6 +1222,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { ...@@ -1184,6 +1222,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
SSmlSTableMeta **tableMeta = SSmlSTableMeta **tableMeta =
(SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
if (tableMeta) { // update meta if (tableMeta) { // update meta
uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf); ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf);
if (ret == TSDB_CODE_SUCCESS) { if (ret == TSDB_CODE_SUCCESS) {
ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf); ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf);
...@@ -1198,7 +1237,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { ...@@ -1198,7 +1237,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
// uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
// return ret; // return ret;
// } // }
uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
if(terrno == TSDB_CODE_DUP_KEY){return terrno;} if(terrno == TSDB_CODE_DUP_KEY){return terrno;}
...@@ -1206,12 +1245,14 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { ...@@ -1206,12 +1245,14 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
} }
} }
uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t smlInsertData(SSmlHandle *info) { static int32_t smlInsertData(SSmlHandle *info) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat);
if(info->pRequest->dbList == NULL){ if(info->pRequest->dbList == NULL){
info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN);
...@@ -1256,6 +1297,7 @@ static int32_t smlInsertData(SSmlHandle *info) { ...@@ -1256,6 +1297,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
// use tablemeta of stable to save vgid and uid of child table // use tablemeta of stable to save vgid and uid of child table
(*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->vgId = vg.vgId;
(*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid
uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat);
code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
...@@ -1278,16 +1320,18 @@ static int32_t smlInsertData(SSmlHandle *info) { ...@@ -1278,16 +1320,18 @@ static int32_t smlInsertData(SSmlHandle *info) {
atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
launchQueryImpl(info->pRequest, info->pQuery, true, NULL); launchQueryImpl(info->pRequest, info->pQuery, true, NULL);
uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code));
return info->pRequest->code; return info->pRequest->code;
} }
static void smlPrintStatisticInfo(SSmlHandle *info) { static void smlPrintStatisticInfo(SSmlHandle *info) {
uDebug( uDebug(
"SML:0x%" PRIx64 "SML:0x%" PRIx64
" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
"", "",
info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables,
info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables,
info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime,
info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime, info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime,
...@@ -1332,6 +1376,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { ...@@ -1332,6 +1376,7 @@ int32_t smlClearForRerun(SSmlHandle *info) {
} }
static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) {
uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (info->protocol == TSDB_SML_JSON_PROTOCOL) { if (info->protocol == TSDB_SML_JSON_PROTOCOL) {
if (lines) { if (lines) {
...@@ -1367,8 +1412,16 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char ...@@ -1367,8 +1412,16 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
} }
} }
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, char cTmp = 0; // for print tmp if is raw
(info->isRawLine ? "rawdata" : tmp)); if(info->isRawLine){
cTmp = tmp[len - 1];
tmp[len - 1] = '\0';
}
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, info->isRawLine, numLines, info->protocol, len, tmp);
if(info->isRawLine){
tmp[len - 1] = cTmp;
}
if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
if (info->dataFormat) { if (info->dataFormat) {
...@@ -1393,6 +1446,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char ...@@ -1393,6 +1446,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
return code; return code;
} }
if (info->reRun) { if (info->reRun) {
uDebug("SML:0x%" PRIx64 " smlParseLine re run", info->id);
i = 0; i = 0;
rawLine = oldRaw; rawLine = oldRaw;
code = smlClearForRerun(info); code = smlClearForRerun(info);
...@@ -1403,6 +1457,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char ...@@ -1403,6 +1457,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
} }
i++; i++;
} }
uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id);
return code; return code;
} }
...@@ -1433,7 +1488,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL ...@@ -1433,7 +1488,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
do { do {
code = smlModifyDBSchemas(info); code = smlModifyDBSchemas(info);
if (code == 0) break; if (code == 0) break;
taosMsleep(200); taosMsleep(500);
uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum);
} while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES);
if (code != 0) { if (code != 0) {
...@@ -1460,6 +1516,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, ...@@ -1460,6 +1516,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
} }
SRequestObj *request = NULL; SRequestObj *request = NULL;
SSmlHandle *info = NULL; SSmlHandle *info = NULL;
int cnt = 0;
while(1){ while(1){
request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid);
if (request == NULL) { if (request == NULL) {
...@@ -1514,16 +1571,22 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, ...@@ -1514,16 +1571,22 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
request->code = code; request->code = code;
info->cost.endTime = taosGetTimestampUs(); info->cost.endTime = taosGetTimestampUs();
info->cost.code = code; info->cost.code = code;
smlPrintStatisticInfo(info); if(code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING
if(code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING){ || code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT){
if(cnt++ >= 10){
uInfo("SML:%"PRIx64" retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
break;
}
taosMsleep(100);
refreshMeta(request->pTscObj, request); refreshMeta(request->pTscObj, request);
uInfo("SML:%"PRIx64" ver is old retry or object is creating code:%d", info->id, code); uInfo("SML:%"PRIx64" retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
smlDestroyInfo(info); smlDestroyInfo(info);
info = NULL; info = NULL;
taos_free_result(request); taos_free_result(request);
request = NULL; request = NULL;
continue; continue;
} }
smlPrintStatisticInfo(info);
break; break;
} }
......
source/client/src/clientSmlJson.c
浏览文件 @ 6811a6b2
...@@ -1237,10 +1237,12 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { ...@@ -1237,10 +1237,12 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) {
if (cnt >= payloadNum) { if (cnt >= payloadNum) {
payloadNum = payloadNum << 1; payloadNum = payloadNum << 1;
void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo));
if (tmp != NULL) { if (tmp == NULL) {
info->lines = (SSmlLineInfo *)tmp; ret = TSDB_CODE_OUT_OF_MEMORY;
memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); return ret;
} }
info->lines = (SSmlLineInfo *)tmp;
memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo));
} }
ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt);
if ((info->lines + cnt)->measure == NULL) break; if ((info->lines + cnt)->measure == NULL) break;
......
source/client/src/clientSmlLine.c
浏览文件 @ 6811a6b2
...@@ -292,6 +292,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin ...@@ -292,6 +292,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
info->currSTableMeta->uid = tinfo->uid; info->currSTableMeta->uid = tinfo->uid;
tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
if (tinfo->tableDataCtx == NULL) { if (tinfo->tableDataCtx == NULL) {
smlDestroyTableInfo(info, tinfo);
smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
...@@ -582,12 +583,14 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine ...@@ -582,12 +583,14 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
.i = ts, .i = ts,
.length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
if (info->dataFormat) { if (info->dataFormat) {
uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);
if(ret != TSDB_CODE_SUCCESS){return ret;} if(ret != TSDB_CODE_SUCCESS){return ret;}
ret = smlBuildRow(info->currTableDataCtx); ret = smlBuildRow(info->currTableDataCtx);
if(ret != TSDB_CODE_SUCCESS){return ret;} if(ret != TSDB_CODE_SUCCESS){return ret;}
clearColValArray(info->currTableDataCtx->pValues); clearColValArray(info->currTableDataCtx->pValues);
} else { } else {
uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts);
taosArraySet(elements->colArray, 0, &kv); taosArraySet(elements->colArray, 0, &kv);
} }
info->preLine = *elements; info->preLine = *elements;
......
source/client/src/clientSmlTelnet.c
浏览文件 @ 6811a6b2
...@@ -292,7 +292,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine ...@@ -292,7 +292,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (info->dataFormat) { if (info->dataFormat && info->currSTableMeta != NULL) {
if (needConverTime) { if (needConverTime) {
kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
} }
......
source/client/src/clientTmq.c
浏览文件 @ 6811a6b2
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
#include "tref.h" #include "tref.h"
#include "ttimer.h" #include "ttimer.h"
#define EMPTY_BLOCK_POLL_IDLE_DURATION 100 #define EMPTY_BLOCK_POLL_IDLE_DURATION 10
#define DEFAULT_AUTO_COMMIT_INTERVAL 5000 #define DEFAULT_AUTO_COMMIT_INTERVAL 5000
struct SMqMgmt { struct SMqMgmt {
...@@ -148,7 +148,8 @@ typedef struct { ...@@ -148,7 +148,8 @@ typedef struct {
typedef struct { typedef struct {
int8_t tmqRspType; int8_t tmqRspType;
int32_t epoch; int32_t epoch; // epoch can be used to guard the vgHandle
int32_t vgId;
SMqClientVg* vgHandle; SMqClientVg* vgHandle;
SMqClientTopic* topicHandle; SMqClientTopic* topicHandle;
uint64_t reqId; uint64_t reqId;
...@@ -580,7 +581,10 @@ static int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, t ...@@ -580,7 +581,10 @@ static int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, t
int32_t code = -1; int32_t code = -1;
taosThreadMutexLock(&tmq->lock); taosThreadMutexLock(&tmq->lock);
for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
tscDebug("consumer:0x%" PRIx64 " user invoked commit offset for %d", tmq->consumerId, numOfTopics);
for (int32_t i = 0; i < numOfTopics; i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (strcmp(pTopic->topicName, topic) != 0) { if (strcmp(pTopic->topicName, topic) != 0) {
continue; continue;
...@@ -874,7 +878,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { ...@@ -874,7 +878,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
return 0; return 0;
} }
static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) {
// do nothing // do nothing
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) {
...@@ -905,6 +909,8 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { ...@@ -905,6 +909,8 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
taosArrayDestroy(pRsp->taosxRsp.createTableLen); taosArrayDestroy(pRsp->taosxRsp.createTableLen);
taosArrayDestroyP(pRsp->taosxRsp.createTableReq, taosMemoryFree); taosArrayDestroyP(pRsp->taosxRsp.createTableReq, taosMemoryFree);
} }
return NULL;
} }
void tmqClearUnhandleMsg(tmq_t* tmq) { void tmqClearUnhandleMsg(tmq_t* tmq) {
...@@ -1324,6 +1330,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { ...@@ -1324,6 +1330,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
pRspWrapper->topicHandle = pTopic; pRspWrapper->topicHandle = pTopic;
pRspWrapper->reqId = requestId; pRspWrapper->reqId = requestId;
pRspWrapper->pEpset = pMsg->pEpSet; pRspWrapper->pEpset = pMsg->pEpSet;
pRspWrapper->vgId = pVg->vgId;
pMsg->pEpSet = NULL; pMsg->pEpSet = NULL;
if (rspType == TMQ_MSG_TYPE__POLL_RSP) { if (rspType == TMQ_MSG_TYPE__POLL_RSP) {
...@@ -1333,10 +1340,10 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { ...@@ -1333,10 +1340,10 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req:%" PRId64 ", rsp:%" PRId64 char buf[80];
" type %d, reqId:0x%" PRIx64, tFormatOffset(buf, 80, &pRspWrapper->dataRsp.rspOffset);
tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version, rspType, requestId); tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64,
tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId);
} else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
...@@ -1375,6 +1382,11 @@ CREATE_MSG_FAIL: ...@@ -1375,6 +1382,11 @@ CREATE_MSG_FAIL:
return -1; return -1;
} }
typedef struct SVgroupSaveInfo {
STqOffsetVal offset;
int64_t numOfRows;
} SVgroupSaveInfo;
static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap,
tmq_t* tmq) { tmq_t* tmq) {
pTopic->schema = pTopicEp->schema; pTopic->schema = pTopicEp->schema;
...@@ -1394,11 +1406,13 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic ...@@ -1394,11 +1406,13 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
makeTopicVgroupKey(vgKey, pTopic->topicName, pVgEp->vgId); makeTopicVgroupKey(vgKey, pTopic->topicName, pVgEp->vgId);
STqOffsetVal* pOffset = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey));
int64_t numOfRows = 0;
STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg}; STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg};
if (pOffset != NULL) { if (pInfo != NULL) {
offsetNew = *pOffset; offsetNew = pInfo->offset;
numOfRows = pInfo->numOfRows;
} }
SMqClientVg clientVg = { SMqClientVg clientVg = {
...@@ -1409,7 +1423,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic ...@@ -1409,7 +1423,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
.vgStatus = TMQ_VG_STATUS__IDLE, .vgStatus = TMQ_VG_STATUS__IDLE,
.vgSkipCnt = 0, .vgSkipCnt = 0,
.emptyBlockReceiveTs = 0, .emptyBlockReceiveTs = 0,
.numOfRows = 0, .numOfRows = numOfRows,
}; };
taosArrayPush(pTopic->vgs, &clientVg); taosArrayPush(pTopic->vgs, &clientVg);
...@@ -1461,7 +1475,9 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { ...@@ -1461,7 +1475,9 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
tFormatOffset(buf, 80, &pVgCur->currentOffset); tFormatOffset(buf, 80, &pVgCur->currentOffset);
tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch,
pVgCur->vgId, vgKey, buf); pVgCur->vgId, vgKey, buf);
taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(STqOffsetVal));
SVgroupSaveInfo info = {.offset = pVgCur->currentOffset, .numOfRows = pVgCur->numOfRows};
taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo));
} }
} }
} }
...@@ -1602,10 +1618,11 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { ...@@ -1602,10 +1618,11 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
return pRspObj; return pRspObj;
} }
SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg) { SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
pRspObj->resType = RES_TYPE__TMQ; pRspObj->resType = RES_TYPE__TMQ;
(*numOfRows) = 0;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
...@@ -1624,8 +1641,8 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg) ...@@ -1624,8 +1641,8 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg)
for(int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { for(int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i); SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i);
int64_t rows = htobe64(pRetrieve->numOfRows); int64_t rows = htobe64(pRetrieve->numOfRows);
pRspObj->resInfo.totalRows += rows;
pVg->numOfRows += rows; pVg->numOfRows += rows;
(*numOfRows) += rows;
} }
return pRspObj; return pRspObj;
...@@ -1732,7 +1749,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { ...@@ -1732,7 +1749,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
for (int j = 0; j < numOfVg; j++) { for (int j = 0; j < numOfVg; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms
tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 100ms before start next poll", tmq->consumerId, tmq->epoch, tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch,
pVg->vgId); pVg->vgId);
continue; continue;
} }
...@@ -1788,29 +1805,28 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { ...@@ -1788,29 +1805,28 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems); tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems);
while (1) { while (1) {
SMqRspWrapper* rspWrapper = NULL; SMqRspWrapper* pRspWrapper = NULL;
taosGetQitem(tmq->qall, (void**)&rspWrapper); taosGetQitem(tmq->qall, (void**)&pRspWrapper);
if (rspWrapper == NULL) { if (pRspWrapper == NULL) {
taosReadAllQitems(tmq->mqueue, tmq->qall); taosReadAllQitems(tmq->mqueue, tmq->qall);
taosGetQitem(tmq->qall, (void**)&rspWrapper); taosGetQitem(tmq->qall, (void**)&pRspWrapper);
if (rspWrapper == NULL) { if (pRspWrapper == NULL) {
return NULL; return NULL;
} }
} }
tscDebug("consumer:0x%"PRIx64" handle rsp, type:%d", tmq->consumerId, rspWrapper->tmqRspType); tscDebug("consumer:0x%"PRIx64" handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType);
if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) {
taosFreeQitem(rspWrapper); taosFreeQitem(pRspWrapper);
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(terrno)); tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(terrno));
return NULL; return NULL;
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
/*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
int32_t consumerEpoch = atomic_load_32(&tmq->epoch); int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp;
...@@ -1833,28 +1849,30 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { ...@@ -1833,28 +1849,30 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
char buf[80]; char buf[80];
tFormatOffset(buf, 80, &pDataRsp->rspOffset); tFormatOffset(buf, 80, &pDataRsp->rspOffset);
if (pDataRsp->blockNum == 0) { if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, reqId:0x%" PRIx64, tmq->consumerId, tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%"PRId64" total:%"PRId64" reqId:0x%" PRIx64, tmq->consumerId,
pVg->vgId, buf, pollRspWrapper->reqId); pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId);
pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
rspWrapper = NULL;
continue;
} else { // build rsp } else { // build rsp
SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg); int64_t numOfRows = 0;
tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%"PRId64" reqId:0x%" PRIx64, SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, pRsp->resInfo.totalRows, pollRspWrapper->reqId); tmq->totalRows += numOfRows;
tmq->totalRows += pRsp->resInfo.totalRows; tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
" vg total:%" PRId64 " total:%" PRId64 ", reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
return pRsp; return pRsp;
} }
} else { } else {
tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
tmq->consumerId, pDataRsp->head.epoch, consumerEpoch); tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch);
tmqFreeRspWrapper(rspWrapper); pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
} }
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch); int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId);
...@@ -1868,13 +1886,13 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { ...@@ -1868,13 +1886,13 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
return pRsp; return pRsp;
} else { } else {
tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
tmq->consumerId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch);
tmqFreeRspWrapper(rspWrapper); pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
} }
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch); int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) {
...@@ -1883,10 +1901,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { ...@@ -1883,10 +1901,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
if (pollRspWrapper->taosxRsp.blockNum == 0) { if (pollRspWrapper->taosxRsp.blockNum == 0) {
rspWrapper = NULL; tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 " reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId);
pollRspWrapper->reqId);
pVg->emptyBlockReceiveTs = taosGetTimestampMs(); pVg->emptyBlockReceiveTs = taosGetTimestampMs();
pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
continue; continue;
} else { } else {
...@@ -1895,32 +1913,37 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { ...@@ -1895,32 +1913,37 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
// build rsp // build rsp
void* pRsp = NULL; void* pRsp = NULL;
int64_t numOfRows = 0;
if (pollRspWrapper->taosxRsp.createTableNum == 0) { if (pollRspWrapper->taosxRsp.createTableNum == 0) {
pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg); pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
} else { } else {
pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper); pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper);
} }
tmq->totalRows += numOfRows;
char buf[80]; char buf[80];
tFormatOffset(buf, 80, &pVg->currentOffset); tFormatOffset(buf, 80, &pVg->currentOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%"PRIx64, tmq->consumerId, pVg->vgId, tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
buf, pollRspWrapper->dataRsp.blockNum, pollRspWrapper->reqId); ", vg total:%" PRId64 " total:%"PRId64" reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows,
tmq->totalRows, pollRspWrapper->reqId);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
return pRsp; return pRsp;
} else { } else {
tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
tmq->consumerId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch);
tmqFreeRspWrapper(rspWrapper); pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
} }
} else { } else {
tscDebug("consumer:0x%" PRIx64 " not data msg received", tmq->consumerId); tscDebug("consumer:0x%" PRIx64 " not data msg received", tmq->consumerId);
bool reset = false; bool reset = false;
tmqHandleNoPollRsp(tmq, rspWrapper, &reset); tmqHandleNoPollRsp(tmq, pRspWrapper, &reset);
taosFreeQitem(rspWrapper); taosFreeQitem(pRspWrapper);
if (pollIfReset && reset) { if (pollIfReset && reset) {
tscDebug("consumer:0x%" PRIx64 ", reset and repoll", tmq->consumerId); tscDebug("consumer:0x%" PRIx64 ", reset and repoll", tmq->consumerId);
tmqPollImpl(tmq, timeout); tmqPollImpl(tmq, timeout);
...@@ -1933,7 +1956,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { ...@@ -1933,7 +1956,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
void* rspObj; void* rspObj;
int64_t startTime = taosGetTimestampMs(); int64_t startTime = taosGetTimestampMs();
tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64, tmq->consumerId, startTime); tscDebug("consumer:0x%" PRIx64 " start to poll at %"PRId64", timeout:%" PRId64, tmq->consumerId, startTime, timeout);
#if 0 #if 0
tmqHandleAllDelayedTask(tmq); tmqHandleAllDelayedTask(tmq);
...@@ -1968,7 +1991,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { ...@@ -1968,7 +1991,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
if (tmqPollImpl(tmq, timeout) < 0) { if (tmqPollImpl(tmq, timeout) < 0) {
tscDebug("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); tscDebug("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId);
/*return NULL;*/
} }
rspObj = tmqHandleAllRsp(tmq, timeout, false); rspObj = tmqHandleAllRsp(tmq, timeout, false);
...@@ -1996,35 +2018,43 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { ...@@ -1996,35 +2018,43 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
} }
} }
int32_t tmq_consumer_close(tmq_t* tmq) { static void displayConsumeStatistics(const tmq_t* pTmq) {
if (tmq->status == TMQ_CONSUMER_STATUS__READY) { int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics);
int32_t rsp = tmq_commit_sync(tmq, NULL); tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d",
if (rsp != 0) { pTmq->consumerId, pTmq->pollCnt, pTmq->totalRows, numOfTopics, pTmq->epoch);
return rsp;
tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", pTmq->consumerId);
for (int32_t i = 0; i < numOfTopics; ++i) {
SMqClientTopic* pTopics = taosArrayGet(pTmq->clientTopics, i);
tscDebug("consumer:0x%" PRIx64 " topic:%d", pTmq->consumerId, i);
int32_t numOfVgs = taosArrayGetSize(pTopics->vgs);
for (int32_t j = 0; j < numOfVgs; ++j) {
SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j);
tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows);
} }
}
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId);
tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", }
tmq->consumerId, tmq->pollCnt, tmq->totalRows, numOfTopics, tmq->epoch);
tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", tmq->consumerId); int32_t tmq_consumer_close(tmq_t* tmq) {
for (int32_t i = 0; i < numOfTopics; ++i) { tscDebug("consumer:0x%" PRIx64" start to close consumer, status:%d", tmq->consumerId, tmq->status);
SMqClientTopic* pTopics = taosArrayGet(tmq->clientTopics, i); displayConsumeStatistics(tmq);
tscDebug("consumer:0x%" PRIx64 " topic:%d", tmq->consumerId, i); if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
int32_t numOfVgs = taosArrayGetSize(pTopics->vgs); // if auto commit is set, commit before close consumer. Otherwise, do nothing.
for (int32_t j = 0; j < numOfVgs; ++j) { if (tmq->autoCommit) {
SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j); int32_t rsp = tmq_commit_sync(tmq, NULL);
tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); if (rsp != 0) {
return rsp;
} }
} }
tscDebug("consumer:0x%" PRIx64 " rows dist end", tmq->consumerId);
int32_t retryCnt = 0; int32_t retryCnt = 0;
tmq_list_t* lst = tmq_list_new(); tmq_list_t* lst = tmq_list_new();
while (1) { while (1) {
rsp = tmq_subscribe(tmq, lst); int32_t rsp = tmq_subscribe(tmq, lst);
if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) {
break; break;
} else { } else {
...@@ -2034,6 +2064,8 @@ int32_t tmq_consumer_close(tmq_t* tmq) { ...@@ -2034,6 +2064,8 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
} }
tmq_list_destroy(lst); tmq_list_destroy(lst);
} else {
tscWarn("consumer:0x%" PRIx64" not in ready state, close it directly", tmq->consumerId);
} }
taosRemoveRef(tmqMgmt.rsetId, tmq->refId); taosRemoveRef(tmqMgmt.rsetId, tmq->refId);
......
source/client/test/clientTests.cpp
浏览文件 @ 6811a6b2
...@@ -112,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) { ...@@ -112,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
} }
taos_free_result(pRes); taos_free_result(pRes);
for(int32_t i = 0; i < 2000; i += 20) { for(int32_t i = 0; i < 100; i += 20) {
char sql[1024] = {0}; char sql[1024] = {0};
sprintf(sql, sprintf(sql,
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
...@@ -167,6 +167,80 @@ void tmq_commit_cb_print(tmq_t *pTmq, int32_t code, void *param) { ...@@ -167,6 +167,80 @@ void tmq_commit_cb_print(tmq_t *pTmq, int32_t code, void *param) {
printf("success, code:%d\n", code); printf("success, code:%d\n", code);
} }
void* doConsumeData(void* param) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
tmq_conf_set(conf, "group.id", "cgrpName12");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "auto.offset.reset", "earliest");
tmq_conf_set(conf, "experimental.snapshot.enable", "true");
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
tmq_conf_destroy(conf);
// 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new();
tmq_list_append(topicList, "topic_t2");
// 启动订阅
tmq_subscribe(tmq, topicList);
tmq_list_destroy(topicList);
TAOS_FIELD* fields = NULL;
int32_t numOfFields = 0;
int32_t precision = 0;
int32_t totalRows = 0;
int32_t msgCnt = 0;
int32_t timeout = 25000;
int32_t count = 0;
while (1) {
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
if (pRes) {
char buf[1024];
const char* topicName = tmq_get_topic_name(pRes);
const char* dbName = tmq_get_db_name(pRes);
int32_t vgroupId = tmq_get_vgroup_id(pRes);
printf("topic: %s\n", topicName);
printf("db: %s\n", dbName);
printf("vgroup id: %d\n", vgroupId);
while (1) {
TAOS_ROW row = taos_fetch_row(pRes);
if (row == NULL) {
break;
}
fields = taos_fetch_fields(pRes);
numOfFields = taos_field_count(pRes);
precision = taos_result_precision(pRes);
taos_print_row(buf, row, fields, numOfFields);
totalRows += 1;
// printf("precision: %d, row content: %s\n", precision, buf);
}
taos_free_result(pRes);
} else {
break;
}
}
tmq_consumer_close(tmq);
taos_close(pConn);
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
return NULL;
}
} // namespace } // namespace
int main(int argc, char** argv) { int main(int argc, char** argv) {
...@@ -188,7 +262,6 @@ TEST(clientCase, driverInit_Test) { ...@@ -188,7 +262,6 @@ TEST(clientCase, driverInit_Test) {
TEST(clientCase, connect_Test) { TEST(clientCase, connect_Test) {
taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) { if (pConn == NULL) {
printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); printf("failed to connect to server, reason:%s\n", taos_errstr(NULL));
...@@ -708,7 +781,7 @@ TEST(clientCase, projection_query_tables) { ...@@ -708,7 +781,7 @@ TEST(clientCase, projection_query_tables) {
// } // }
// taos_free_result(pRes); // taos_free_result(pRes);
TAOS_RES* pRes = taos_query(pConn, "use abc2"); TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
...@@ -730,7 +803,7 @@ TEST(clientCase, projection_query_tables) { ...@@ -730,7 +803,7 @@ TEST(clientCase, projection_query_tables) {
} }
taos_free_result(pRes); taos_free_result(pRes);
for (int32_t i = 0; i < 2; ++i) { for (int32_t i = 0; i < 10000; ++i) {
printf("create table :%d\n", i); printf("create table :%d\n", i);
createNewTable(pConn, i); createNewTable(pConn, i);
} }
...@@ -970,28 +1043,23 @@ TEST(clientCase, sub_db_test) { ...@@ -970,28 +1043,23 @@ TEST(clientCase, sub_db_test) {
taos_print_row(buf, row, fields, numOfFields); taos_print_row(buf, row, fields, numOfFields);
printf("precision: %d, row content: %s\n", precision, buf); printf("precision: %d, row content: %s\n", precision, buf);
} }
taos_free_result(pRes);
} }
// return rows;
} }
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
} }
TEST(clientCase, sub_tb_test) { TEST(clientCase, sub_tb_test) {
taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
// TAOS_RES* pRes = taos_query(pConn, "create topic topic_t1 as select * from t1");
// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
// printf("failed to create topic, code:%s", taos_errstr(pRes));
// taos_free_result(pRes);
// return;
// }
tmq_conf_t* conf = tmq_conf_new(); tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
tmq_conf_set(conf, "group.id", "cgrpName"); tmq_conf_set(conf, "group.id", "cgrpName27");
tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_conf_set(conf, "auto.offset.reset", "earliest");
...@@ -1004,10 +1072,11 @@ TEST(clientCase, sub_tb_test) { ...@@ -1004,10 +1072,11 @@ TEST(clientCase, sub_tb_test) {
// 创建订阅 topics 列表 // 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new(); tmq_list_t* topicList = tmq_list_new();
tmq_list_append(topicList, "topic_t1"); tmq_list_append(topicList, "topic_t2");
// 启动订阅 // 启动订阅
tmq_subscribe(tmq, topicList); tmq_subscribe(tmq, topicList);
tmq_list_destroy(topicList); tmq_list_destroy(topicList);
TAOS_FIELD* fields = NULL; TAOS_FIELD* fields = NULL;
...@@ -1015,7 +1084,7 @@ TEST(clientCase, sub_tb_test) { ...@@ -1015,7 +1084,7 @@ TEST(clientCase, sub_tb_test) {
int32_t precision = 0; int32_t precision = 0;
int32_t totalRows = 0; int32_t totalRows = 0;
int32_t msgCnt = 0; int32_t msgCnt = 0;
int32_t timeout = 5000; int32_t timeout = 25000;
int32_t count = 0; int32_t count = 0;
...@@ -1023,7 +1092,6 @@ TEST(clientCase, sub_tb_test) { ...@@ -1023,7 +1092,6 @@ TEST(clientCase, sub_tb_test) {
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
if (pRes) { if (pRes) {
char buf[1024]; char buf[1024];
int32_t rows = 0;
const char* topicName = tmq_get_topic_name(pRes); const char* topicName = tmq_get_topic_name(pRes);
const char* dbName = tmq_get_db_name(pRes); const char* dbName = tmq_get_db_name(pRes);
...@@ -1033,27 +1101,45 @@ TEST(clientCase, sub_tb_test) { ...@@ -1033,27 +1101,45 @@ TEST(clientCase, sub_tb_test) {
printf("db: %s\n", dbName); printf("db: %s\n", dbName);
printf("vgroup id: %d\n", vgroupId); printf("vgroup id: %d\n", vgroupId);
if (count ++ > 200) {
tmq_unsubscribe(tmq);
break;
}
while (1) { while (1) {
TAOS_ROW row = taos_fetch_row(pRes); TAOS_ROW row = taos_fetch_row(pRes);
if (row == NULL) break; if (row == NULL) {
break;
}
fields = taos_fetch_fields(pRes); fields = taos_fetch_fields(pRes);
numOfFields = taos_field_count(pRes); numOfFields = taos_field_count(pRes);
precision = taos_result_precision(pRes); precision = taos_result_precision(pRes);
rows++;
taos_print_row(buf, row, fields, numOfFields); taos_print_row(buf, row, fields, numOfFields);
totalRows += 1;
printf("precision: %d, row content: %s\n", precision, buf); printf("precision: %d, row content: %s\n", precision, buf);
} }
taos_free_result(pRes);
// if ((++count) > 1) {
// break;
// }
} else {
break;
} }
// return rows;
} }
tmq_consumer_close(tmq);
taos_close(pConn);
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
} }
TEST(clientCase, sub_tb_mt_test) {
taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
TdThread qid[20] = {0};
for(int32_t i = 0; i < 1; ++i) {
taosThreadCreate(&qid[i], NULL, doConsumeData, NULL);
}
for(int32_t i = 0; i < 4; ++i) {
taosThreadJoin(qid[i], NULL);
}
}
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
source/common/src/systable.c
浏览文件 @ 6811a6b2
...@@ -233,7 +233,7 @@ static const SSysDbTableSchema vgroupsSchema[] = { ...@@ -233,7 +233,7 @@ static const SSysDbTableSchema vgroupsSchema[] = {
{.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "cacheTables", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "cacheelements", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
// {.name = "compact_start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, // {.name = "compact_start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
}; };
......
source/common/src/tglobal.c
浏览文件 @ 6811a6b2
...@@ -1238,13 +1238,13 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi ...@@ -1238,13 +1238,13 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
} }
if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr()); printf("failed to load cfg since %s", terrstr());
cfgCleanup(pCfg); cfgCleanup(pCfg);
return -1; return -1;
} }
if (cfgLoadFromArray(pCfg, pArgs) != 0) { if (cfgLoadFromArray(pCfg, pArgs) != 0) {
uError("failed to load cfg from array since %s", terrstr()); printf("failed to load cfg from array since %s", terrstr());
cfgCleanup(pCfg); cfgCleanup(pCfg);
return -1; return -1;
} }
...@@ -1260,13 +1260,13 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi ...@@ -1260,13 +1260,13 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
if (taosMulModeMkDir(tsLogDir, 0777) != 0) { if (taosMulModeMkDir(tsLogDir, 0777) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to create dir:%s since %s", tsLogDir, terrstr()); printf("failed to create dir:%s since %s", tsLogDir, terrstr());
cfgCleanup(pCfg); cfgCleanup(pCfg);
return -1; return -1;
} }
if (taosInitLog(logname, logFileNum) != 0) { if (taosInitLog(logname, logFileNum) != 0) {
uError("failed to init log file since %s", terrstr()); printf("failed to init log file since %s", terrstr());
cfgCleanup(pCfg); cfgCleanup(pCfg);
return -1; return -1;
} }
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ 6811a6b2
...@@ -457,6 +457,7 @@ typedef struct { ...@@ -457,6 +457,7 @@ typedef struct {
void* pIter; void* pIter;
SMnode* pMnode; SMnode* pMnode;
STableMetaRsp* pMeta; STableMetaRsp* pMeta;
bool restore;
bool sysDbRsp; bool sysDbRsp;
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
char filterTb[TSDB_TABLE_NAME_LEN]; char filterTb[TSDB_TABLE_NAME_LEN];
......
source/dnode/mnode/impl/src/mndMain.c
浏览文件 @ 6811a6b2
...@@ -864,7 +864,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr ...@@ -864,7 +864,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
} }
// grant info // grant info
pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f; pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 1000;
pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;
if (pMnode->grant.expireTimeMS == 0) { if (pMnode->grant.expireTimeMS == 0) {
pGrantInfo->expire_time = INT32_MAX; pGrantInfo->expire_time = INT32_MAX;
......
source/dnode/mnode/impl/src/mndScheduler.c
浏览文件 @ 6811a6b2
...@@ -115,7 +115,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream ...@@ -115,7 +115,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream
if (pStream->fixedSinkVgId == 0) { if (pStream->fixedSinkVgId == 0) {
SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb);
if (pDb->cfg.numOfVgroups > 1) { if (pDb != NULL && pDb->cfg.numOfVgroups > 1) {
isShuffle = true; isShuffle = true;
pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH; pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH;
pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
......
source/dnode/mnode/impl/src/mndShow.c
浏览文件 @ 6811a6b2
...@@ -134,7 +134,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { ...@@ -134,7 +134,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) {
showObj.pMnode = pMnode; showObj.pMnode = pMnode;
showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb));
memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN);
strncpy(showObj.filterTb, pReq->filterTb, TSDB_TABLE_NAME_LEN); tstrncpy(showObj.filterTb, pReq->filterTb, TSDB_TABLE_NAME_LEN);
int32_t keepTime = tsShellActivityTimer * 6 * 1000; int32_t keepTime = tsShellActivityTimer * 6 * 1000;
SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime);
...@@ -324,7 +324,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { ...@@ -324,7 +324,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
pReq->info.rsp = pRsp; pReq->info.rsp = pRsp;
pReq->info.rspLen = size; pReq->info.rspLen = size;
if (rowsRead == 0 || rowsRead < rowsToRead) { if (rowsRead == 0 || ((rowsRead < rowsToRead) && !pShow->restore)) {
pRsp->completed = 1; pRsp->completed = 1;
mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id);
mndReleaseShowObj(pShow, true); mndReleaseShowObj(pShow, true);
......
source/dnode/mnode/impl/src/mndStb.c
浏览文件 @ 6811a6b2
...@@ -3113,22 +3113,24 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB ...@@ -3113,22 +3113,24 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(typeName, "SUPER_TABLE"); STR_TO_VARSTR(typeName, "SUPER_TABLE");
bool fetch = pShow->restore ? false : true;
pShow->restore = false;
while (numOfRows < rows) { while (numOfRows < rows) {
void *prevIter = pShow->pIter; if (fetch) {
pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb);
if (pShow->pIter == NULL) break; if (pShow->pIter == NULL) break;
} else {
fetch = true;
void *pKey = taosHashGetKey(pShow->pIter, NULL);
pStb = sdbAcquire(pSdb, SDB_STB, pKey);
if (!pStb) continue;
}
if (pDb != NULL && pStb->dbUid != pDb->uid) { if (pDb != NULL && pStb->dbUid != pDb->uid) {
sdbRelease(pSdb, pStb); sdbRelease(pSdb, pStb);
continue; continue;
} }
if ((numOfRows + pStb->numOfColumns) > rows) {
pShow->pIter = prevIter;
sdbRelease(pSdb, pStb);
break;
}
SName name = {0}; SName name = {0};
char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN);
...@@ -3136,6 +3138,17 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB ...@@ -3136,6 +3138,17 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
sdbRelease(pSdb, pStb); sdbRelease(pSdb, pStb);
continue; continue;
} }
if ((numOfRows + pStb->numOfColumns) > rows) {
pShow->restore = true;
if (numOfRows == 0) {
mError("mndRetrieveStbCol failed to get stable cols since buf:%d less than result:%d, stable name:%s, db:%s",
rows, pStb->numOfColumns, pStb->name, pStb->db);
}
sdbRelease(pSdb, pStb);
break;
}
varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE]));
mDebug("mndRetrieveStbCol get stable cols, stable name:%s, db:%s", pStb->name, pStb->db); mDebug("mndRetrieveStbCol get stable cols, stable name:%s, db:%s", pStb->name, pStb->db);
......
source/dnode/snode/src/snode.c
浏览文件 @ 6811a6b2
...@@ -75,9 +75,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { ...@@ -75,9 +75,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
pTask->pMsgCb = &pSnode->msgCb; pTask->pMsgCb = &pSnode->msgCb;
pTask->startVer = ver; pTask->startVer = ver;
pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1);
...@@ -90,11 +88,11 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { ...@@ -90,11 +88,11 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
.numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo),
.pStateBackend = pTask->pState, .pStateBackend = pTask->pState,
}; };
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle);
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0);
ASSERT(pTask->exec.executor); ASSERT(pTask->exec.executor);
streamSetupTrigger(pTask); streamSetupTrigger(pTask);
return 0; return 0;
} }
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ 6811a6b2
...@@ -157,7 +157,7 @@ typedef struct SMTbCursor SMTbCursor; ...@@ -157,7 +157,7 @@ typedef struct SMTbCursor SMTbCursor;
SMTbCursor *metaOpenTbCursor(SMeta *pMeta); SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
void metaCloseTbCursor(SMTbCursor *pTbCur); void metaCloseTbCursor(SMTbCursor *pTbCur);
int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType);
int32_t metaTbCursorPrev(SMTbCursor *pTbCur); int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType);
#endif #endif
...@@ -180,6 +180,7 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t n ...@@ -180,6 +180,7 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t n
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr);
void tsdbReaderSetId(STsdbReader* pReader, const char* idstr);
void tsdbReaderClose(STsdbReader *pReader); void tsdbReaderClose(STsdbReader *pReader);
bool tsdbNextDataBlock(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader);
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
......
source/dnode/vnode/src/inc/tq.h
浏览文件 @ 6811a6b2
...@@ -141,7 +141,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); ...@@ -141,7 +141,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
// tqRead // tqRead
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset); int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset);
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum, uint64_t reqId);
// tqExec // tqExec
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp); int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp);
......
source/dnode/vnode/src/meta/metaQuery.c
浏览文件 @ 6811a6b2
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
*/ */
#include "meta.h" #include "meta.h"
#include "osMemory.h"
#include "tencode.h"
void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) {
memset(pReader, 0, sizeof(*pReader)); memset(pReader, 0, sizeof(*pReader));
...@@ -334,7 +336,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { ...@@ -334,7 +336,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) {
return 0; return 0;
} }
int32_t metaTbCursorPrev(SMTbCursor *pTbCur) { int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
int ret; int ret;
void *pBuf; void *pBuf;
STbCfg tbCfg; STbCfg tbCfg;
...@@ -348,7 +350,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur) { ...@@ -348,7 +350,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur) {
tDecoderClear(&pTbCur->mr.coder); tDecoderClear(&pTbCur->mr.coder);
metaGetTableEntryByVersion(&pTbCur->mr, ((SUidIdxVal *)pTbCur->pVal)[0].version, *(tb_uid_t *)pTbCur->pKey); metaGetTableEntryByVersion(&pTbCur->mr, ((SUidIdxVal *)pTbCur->pVal)[0].version, *(tb_uid_t *)pTbCur->pKey);
if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) { if (pTbCur->mr.me.type == jumpTableType) {
continue; continue;
} }
...@@ -1235,9 +1237,14 @@ END: ...@@ -1235,9 +1237,14 @@ END:
return 0; return 0;
} }
int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
int32_t ret = 0; SMetaEntry oStbEntry = {0};
char *buf = NULL; int32_t ret = -1;
char *buf = NULL;
void *pData = NULL;
int nData = 0;
SDecoder dc = {0};
STbDbKey tbDbKey = {0};
STagIdxKey *pKey = NULL; STagIdxKey *pKey = NULL;
int32_t nKey = 0; int32_t nKey = 0;
...@@ -1249,8 +1256,34 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { ...@@ -1249,8 +1256,34 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
pCursor->type = param->type; pCursor->type = param->type;
metaRLock(pMeta); metaRLock(pMeta);
if (tdbTbGet(pMeta->pUidIdx, &param->suid, sizeof(tb_uid_t), &pData, &nData) != 0) {
goto END;
}
tbDbKey.uid = param->suid;
tbDbKey.version = ((SUidIdxVal *)pData)[0].version;
tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData);
tDecoderInit(&dc, pData, nData);
ret = metaDecodeEntry(&dc, &oStbEntry);
if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) {
ret = -1;
goto END;
}
ret = -1;
for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) {
SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i;
if (schema->colId == param->cid && param->type == schema->type && (IS_IDX_ON(schema) || i == 0)) {
ret = 0;
}
}
if (ret != 0) {
goto END;
}
ret = tdbTbcOpen(pMeta->pTagIdx, &pCursor->pCur, NULL); ret = tdbTbcOpen(pMeta->pTagIdx, &pCursor->pCur, NULL);
if (ret < 0) { if (ret != 0) {
goto END; goto END;
} }
...@@ -1271,6 +1304,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { ...@@ -1271,6 +1304,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
maxSize = 4 * nTagData + 1; maxSize = 4 * nTagData + 1;
buf = taosMemoryCalloc(1, maxSize); buf = taosMemoryCalloc(1, maxSize);
if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize)) { if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize)) {
ret = -1;
goto END; goto END;
} }
...@@ -1288,8 +1322,10 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { ...@@ -1288,8 +1322,10 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
if (ret != 0) { if (ret != 0) {
goto END; goto END;
} }
int cmp = 0; int cmp = 0;
if (tdbTbcMoveTo(pCursor->pCur, pKey, nKey, &cmp) < 0) { ret = tdbTbcMoveTo(pCursor->pCur, pKey, nKey, &cmp);
if (ret != 0) {
goto END; goto END;
} }
...@@ -1353,6 +1389,10 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { ...@@ -1353,6 +1389,10 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
END: END:
if (pCursor->pMeta) metaULock(pCursor->pMeta); if (pCursor->pMeta) metaULock(pCursor->pMeta);
if (pCursor->pCur) tdbTbcClose(pCursor->pCur); if (pCursor->pCur) tdbTbcClose(pCursor->pCur);
if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
tDecoderClear(&dc);
tdbFree(pData);
taosMemoryFree(buf); taosMemoryFree(buf);
taosMemoryFree(pKey); taosMemoryFree(pKey);
......
source/dnode/vnode/src/sma/smaRollup.c
浏览文件 @ 6811a6b2
...@@ -282,7 +282,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat ...@@ -282,7 +282,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
.initTqReader = 1, .initTqReader = 1,
.pStateBackend = pStreamState, .pStateBackend = pStreamState,
}; };
pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode));
if (!pRSmaInfo->taskInfo[idx]) { if (!pRSmaInfo->taskInfo[idx]) {
terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
...@@ -864,7 +864,7 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t ...@@ -864,7 +864,7 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode));
if (!dstTaskInfo) { if (!dstTaskInfo) {
code = TSDB_CODE_RSMA_QTASKINFO_CREATE; code = TSDB_CODE_RSMA_QTASKINFO_CREATE;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
......
source/dnode/vnode/src/tq/tq.c
浏览文件 @ 6811a6b2
...@@ -296,51 +296,6 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con ...@@ -296,51 +296,6 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
} }
} }
#endif #endif
// int32_t len = 0;
// int32_t code = 0;
//
// if (type == TMQ_MSG_TYPE__POLL_RSP) {
// tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
// } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
// tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code);
// }
//
// if (code < 0) {
// return -1;
// }
//
// int32_t tlen = sizeof(SMqRspHead) + len;
// void* buf = rpcMallocCont(tlen);
// if (buf == NULL) {
// return -1;
// }
//
// ((SMqRspHead*)buf)->mqMsgType = type;
// ((SMqRspHead*)buf)->epoch = pReq->epoch;
// ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
//
// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
//
// SEncoder encoder = {0};
// tEncoderInit(&encoder, abuf, len);
//
// if (type == TMQ_MSG_TYPE__POLL_RSP) {
// tEncodeSMqDataRsp(&encoder, pRsp);
// } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
// tEncodeSTaosxRsp(&encoder, (STaosxRsp*) pRsp);
// }
//
// tEncoderClear(&encoder);
//
// SRpcMsg rsp = {
// .info = pMsg->info,
// .pCont = buf,
// .contLen = tlen,
// .code = 0,
// };
//
// tmsgSendRsp(&rsp);
doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type); doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type);
char buf1[80] = {0}; char buf1[80] = {0};
...@@ -348,76 +303,12 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con ...@@ -348,76 +303,12 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf1, 80, &pRsp->reqOffset);
tFormatOffset(buf2, 80, &pRsp->rspOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset);
tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s", tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%"PRIx64,
TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2); TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId);
return 0; return 0;
} }
//int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) {
//#if 0
// A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
// A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
//
// if (pRsp->withSchema) {
// A(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum);
// } else {
// A(taosArrayGetSize(pRsp->blockSchema) == 0);
// }
//
// if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
// if (pRsp->blockNum > 0) {
// A(pRsp->rspOffset.version > pRsp->reqOffset.version);
// } else {
// A(pRsp->rspOffset.version >= pRsp->reqOffset.version);
// }
// }
//#endif
//
// int32_t len = 0;
// int32_t code = 0;
// tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code);
// if (code < 0) {
// return -1;
// }
//
// int32_t tlen = sizeof(SMqRspHead) + len;
// void* buf = rpcMallocCont(tlen);
// if (buf == NULL) {
// terrno = TSDB_CODE_OUT_OF_MEMORY;
// return -1;
// }
//
// ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__TAOSX_RSP;
// ((SMqRspHead*)buf)->epoch = pReq->epoch;
// ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
//
// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
//
// SEncoder encoder = {0};
// tEncoderInit(&encoder, abuf, len);
// tEncodeSTaosxRsp(&encoder, pRsp);
// tEncoderClear(&encoder);
//
// SRpcMsg rsp = {
// .info = pMsg->info,
// .pCont = buf,
// .contLen = tlen,
// .code = 0,
// };
//
// tmsgSendRsp(&rsp);
//
// char buf1[80] = {0};
// char buf2[80] = {0};
// tFormatOffset(buf1, 80, &pRsp->reqOffset);
// tFormatOffset(buf2, 80, &pRsp->rspOffset);
//
// tqDebug("taosx rsp, vgId:%d, consumer:0x%" PRIx64 " (epoch %d) send rsp, numOfBlks:%d, req:%s, rsp:%s",
// TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
// return 0;
//}
static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) {
return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG &&
pLeft->val.version <= pRight->val.version; pLeft->val.version <= pRight->val.version;
...@@ -425,6 +316,7 @@ static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOf ...@@ -425,6 +316,7 @@ static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOf
int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
STqOffset offset = {0}; STqOffset offset = {0};
int32_t vgId = TD_VID(pTq->pVnode);
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msg, msgLen); tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
...@@ -436,10 +328,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t ...@@ -436,10 +328,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA || offset.val.type == TMQ_OFFSET__SNAPSHOT_META) { if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA || offset.val.type == TMQ_OFFSET__SNAPSHOT_META) {
tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64,
offset.subKey, TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); offset.subKey, vgId, offset.val.uid, offset.val.ts);
} else if (offset.val.type == TMQ_OFFSET__LOG) { } else if (offset.val.type == TMQ_OFFSET__LOG) {
tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey, tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey,
TD_VID(pTq->pVnode), offset.val.version); vgId, offset.val.version);
if (offset.val.version + 1 == sversion) { if (offset.val.version + 1 == sversion) {
offset.val.version += 1; offset.val.version += 1;
} }
...@@ -542,6 +434,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand ...@@ -542,6 +434,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
uint64_t consumerId = pRequest->consumerId; uint64_t consumerId = pRequest->consumerId;
STqOffsetVal reqOffset = pRequest->reqOffset; STqOffsetVal reqOffset = pRequest->reqOffset;
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey);
int32_t vgId = TD_VID(pTq->pVnode);
*pBlockReturned = false; *pBlockReturned = false;
// In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value.
...@@ -551,12 +445,15 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand ...@@ -551,12 +445,15 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
char formatBuf[80]; char formatBuf[80];
tFormatOffset(formatBuf, 80, pOffsetVal); tFormatOffset(formatBuf, 80, pOffsetVal);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, prev offset found, offset reset to %s and continue.", tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, prev offset found, offset reset to %s and continue.",
consumerId, pHandle->subKey, TD_VID(pTq->pVnode), formatBuf); consumerId, pHandle->subKey, vgId, formatBuf);
return 0; return 0;
} else { } else {
// no poll occurs in this vnode for this topic, let's seek to the right offset value. // no poll occurs in this vnode for this topic, let's seek to the right offset value.
if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
if (pRequest->useSnapshot) { if (pRequest->useSnapshot) {
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot",
consumerId, pHandle->subKey, vgId);
if (pHandle->fetchMeta) { if (pHandle->fetchMeta) {
tqOffsetResetToMeta(pOffsetVal, 0); tqOffsetResetToMeta(pOffsetVal, 0);
} else { } else {
...@@ -577,8 +474,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand ...@@ -577,8 +474,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, offset reset to %" PRId64, consumerId, tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version); pHandle->subKey, vgId, dataRsp.rspOffset.version);
int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
tDeleteSMqDataRsp(&dataRsp); tDeleteSMqDataRsp(&dataRsp);
...@@ -589,16 +486,14 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand ...@@ -589,16 +486,14 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
tqInitTaosxRsp(&taosxRsp, pRequest); tqInitTaosxRsp(&taosxRsp, pRequest);
tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
// int32_t code = tqSendTaosxRsp(pTq, pMsg, pRequest, &taosxRsp);
tDeleteSTaosxRsp(&taosxRsp); tDeleteSTaosxRsp(&taosxRsp);
*pBlockReturned = true; *pBlockReturned = true;
return code; return code;
} }
} else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
tqError("tmq poll: subkey %s, no offset committed for consumer:0x%" PRIx64 tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed",
" in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey);
pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pRequest->subKey);
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
return -1; return -1;
} }
...@@ -639,6 +534,8 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* ...@@ -639,6 +534,8 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
// lock // lock
taosWLockLatch(&pTq->pushLock); taosWLockLatch(&pTq->pushLock);
qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
code = tqScanData(pTq, pHandle, &dataRsp, &offset); code = tqScanData(pTq, pHandle, &dataRsp, &offset);
// till now, all data has been transferred to consumer, new data needs to push client once arrived. // till now, all data has been transferred to consumer, new data needs to push client once arrived.
...@@ -654,9 +551,9 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* ...@@ -654,9 +551,9 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
// NOTE: this pHandle->consumerId may have been changed already. // NOTE: this pHandle->consumerId may have been changed already.
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64 tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64
", ts:%" PRId64, ", ts:%" PRId64", reqId:0x%"PRIx64,
consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid, consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid,
dataRsp.rspOffset.ts); dataRsp.rspOffset.ts, pRequest->reqId);
tDeleteSMqDataRsp(&dataRsp); tDeleteSMqDataRsp(&dataRsp);
return code; return code;
...@@ -676,9 +573,9 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* ...@@ -676,9 +573,9 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
if (metaRsp.metaRspLen > 0) { if (metaRsp.metaRspLen > 0) {
code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp); code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp);
tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64
",version:%" PRId64, ",ts:%" PRId64,
consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
metaRsp.rspOffset.version); metaRsp.rspOffset.ts);
taosMemoryFree(metaRsp.metaRsp); taosMemoryFree(metaRsp.metaRsp);
tDeleteSTaosxRsp(&taosxRsp); tDeleteSTaosxRsp(&taosxRsp);
return code; return code;
...@@ -719,17 +616,12 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* ...@@ -719,17 +616,12 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
break; break;
} }
if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) { if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
// if (terrno == 0) { // failed to seek to given ver, but no errors happen. code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
// code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, (SMqDataRsp*) &taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); tDeleteSTaosxRsp(&taosxRsp);
// return code; taosMemoryFreeClear(pCkHead);
// } else { // error happens, return to consumers return code;
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
tDeleteSTaosxRsp(&taosxRsp);
taosMemoryFreeClear(pCkHead);
return code;
// }
} }
SWalCont* pHead = &pCkHead->head; SWalCont* pHead = &pCkHead->head;
...@@ -906,7 +798,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ...@@ -906,7 +798,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
SMqRebVgReq req = {0}; SMqRebVgReq req = {0};
tDecodeSMqRebVgReq(msg, &req); tDecodeSMqRebVgReq(msg, &req);
tqDebug("vgId:%d, tq process sub req %s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, SVnode* pVnode = pTq->pVnode;
int32_t vgId = TD_VID(pVnode);
tqDebug("vgId:%d, tq process sub req %s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey,
req.oldConsumerId, req.newConsumerId); req.oldConsumerId, req.newConsumerId);
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
...@@ -935,7 +830,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ...@@ -935,7 +830,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
pHandle->fetchMeta = req.withMeta; pHandle->fetchMeta = req.withMeta;
// TODO version should be assigned and refed during preprocess // TODO version should be assigned and refed during preprocess
SWalRef* pRef = walRefCommittedVer(pTq->pVnode->pWal); SWalRef* pRef = walRefCommittedVer(pVnode->pWal);
if (pRef == NULL) { if (pRef == NULL) {
taosMemoryFree(req.qmsg); taosMemoryFree(req.qmsg);
return -1; return -1;
...@@ -945,8 +840,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ...@@ -945,8 +840,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
pHandle->pRef = pRef; pHandle->pRef = pRef;
SReadHandle handle = { SReadHandle handle = {
.meta = pTq->pVnode->pMeta, .meta = pVnode->pMeta,
.vnode = pTq->pVnode, .vnode = pVnode,
.initTableReader = true, .initTableReader = true,
.initTqReader = true, .initTqReader = true,
.version = ver, .version = ver,
...@@ -959,38 +854,38 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ...@@ -959,38 +854,38 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
req.qmsg = NULL; req.qmsg = NULL;
pHandle->execHandle.task = pHandle->execHandle.task =
qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, &pHandle->execHandle.numOfCols, NULL); qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, NULL);
void* scanner = NULL; void* scanner = NULL;
qExtractStreamScanner(pHandle->execHandle.task, &scanner); qExtractStreamScanner(pHandle->execHandle.task, &scanner);
pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner);
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL);
pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode); pHandle->execHandle.pExecReader = tqOpenReader(pVnode);
pHandle->execHandle.execDb.pFilterOutTbUid = pHandle->execHandle.execDb.pFilterOutTbUid =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
(SSnapContext**)(&handle.sContext)); (SSnapContext**)(&handle.sContext));
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL); pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, NULL);
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL);
pHandle->execHandle.execTb.suid = req.suid; pHandle->execHandle.execTb.suid = req.suid;
SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList); vnodeGetCtbIdList(pVnode, req.suid, tbUidList);
tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, req.suid); tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pVnode->config.vgId, req.suid);
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid);
} }
pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode); pHandle->execHandle.pExecReader = tqOpenReader(pVnode);
tqReaderSetTbUidList(pHandle->execHandle.pExecReader, tbUidList); tqReaderSetTbUidList(pHandle->execHandle.pExecReader, tbUidList);
taosArrayDestroy(tbUidList); taosArrayDestroy(tbUidList);
buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
(SSnapContext**)(&handle.sContext)); (SSnapContext**)(&handle.sContext));
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL); pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, NULL);
} }
taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
...@@ -1043,6 +938,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { ...@@ -1043,6 +938,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
} }
#endif #endif
int32_t vgId = TD_VID(pTq->pVnode);
pTask->refCnt = 1; pTask->refCnt = 1;
pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
...@@ -1055,9 +951,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { ...@@ -1055,9 +951,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
pTask->pMsgCb = &pTq->pVnode->msgCb; pTask->pMsgCb = &pTq->pVnode->msgCb;
pTask->startVer = ver; pTask->startVer = ver;
// expand executor // expand executor
...@@ -1077,7 +971,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { ...@@ -1077,7 +971,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
.initTqReader = 1, .initTqReader = 1,
.pStateBackend = pTask->pState, .pStateBackend = pTask->pState,
}; };
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId);
if (pTask->exec.executor == NULL) { if (pTask->exec.executor == NULL) {
return -1; return -1;
} }
...@@ -1092,7 +987,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { ...@@ -1092,7 +987,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
.numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo),
.pStateBackend = pTask->pState, .pStateBackend = pTask->pState,
}; };
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle);
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId);
if (pTask->exec.executor == NULL) { if (pTask->exec.executor == NULL) {
return -1; return -1;
} }
...@@ -1122,9 +1018,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { ...@@ -1122,9 +1018,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
} }
streamSetupTrigger(pTask); streamSetupTrigger(pTask);
tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", vgId, pTask->taskId, pTask->selfChildId, pTask->taskLevel);
tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", TD_VID(pTq->pVnode), pTask->taskId,
pTask->selfChildId, pTask->taskLevel);
return 0; return 0;
} }
......
source/dnode/vnode/src/tq/tqExec.c
浏览文件 @ 6811a6b2
...@@ -92,7 +92,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs ...@@ -92,7 +92,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
return -1; return -1;
} }
tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock); tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task executed, get %p", pHandle->consumerId, pTq->pVnode->config.vgId, pDataBlock);
// current scan should be stopped asap, since the rebalance occurs. // current scan should be stopped asap, since the rebalance occurs.
if (pDataBlock == NULL) { if (pDataBlock == NULL) {
...@@ -120,7 +120,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs ...@@ -120,7 +120,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
return -1; return -1;
} }
if(pRsp->withTbName || pRsp->withSchema){ if (pRsp->withTbName || pRsp->withSchema) {
tqError("get column should not with meta:%d,%d", pRsp->withTbName, pRsp->withSchema); tqError("get column should not with meta:%d,%d", pRsp->withTbName, pRsp->withSchema);
return -1; return -1;
} }
......
source/dnode/vnode/src/tq/tqMeta.c
浏览文件 @ 6811a6b2
...@@ -269,11 +269,13 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { ...@@ -269,11 +269,13 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) {
} }
int32_t tqMetaRestoreHandle(STQ* pTq) { int32_t tqMetaRestoreHandle(STQ* pTq) {
int code = 0;
TBC* pCur = NULL; TBC* pCur = NULL;
if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) { if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) {
return -1; return -1;
} }
int32_t vgId = TD_VID(pTq->pVnode);
void* pKey = NULL; void* pKey = NULL;
int kLen = 0; int kLen = 0;
void* pVal = NULL; void* pVal = NULL;
...@@ -290,7 +292,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { ...@@ -290,7 +292,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
handle.pRef = walOpenRef(pTq->pVnode->pWal); handle.pRef = walOpenRef(pTq->pVnode->pWal);
if (handle.pRef == NULL) { if (handle.pRef == NULL) {
return -1; code = -1;
goto end;
} }
walRefVer(handle.pRef, handle.snapshotVer); walRefVer(handle.pRef, handle.snapshotVer);
...@@ -304,19 +307,24 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { ...@@ -304,19 +307,24 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
handle.execHandle.task = handle.execHandle.task =
qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, NULL); qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, NULL);
if (handle.execHandle.task == NULL) { if (handle.execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle.subKey); tqError("cannot create exec task for %s", handle.subKey);
return -1; code = -1;
goto end;
} }
void* scanner = NULL; void* scanner = NULL;
qExtractStreamScanner(handle.execHandle.task, &scanner); qExtractStreamScanner(handle.execHandle.task, &scanner);
if (scanner == NULL) { if (scanner == NULL) {
tqError("cannot extract stream scanner for %s", handle.subKey); tqError("cannot extract stream scanner for %s", handle.subKey);
code = -1;
goto end;
} }
handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner);
if (handle.execHandle.pExecReader == NULL) { if (handle.execHandle.pExecReader == NULL) {
tqError("cannot extract exec reader for %s", handle.subKey); tqError("cannot extract exec reader for %s", handle.subKey);
code = -1;
goto end;
} }
} else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) {
handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
...@@ -324,7 +332,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { ...@@ -324,7 +332,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta,
(SSnapContext**)(&reader.sContext)); (SSnapContext**)(&reader.sContext));
handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, NULL, NULL); handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, NULL);
} else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
...@@ -341,14 +349,15 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { ...@@ -341,14 +349,15 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType,
handle.fetchMeta, (SSnapContext**)(&reader.sContext)); handle.fetchMeta, (SSnapContext**)(&reader.sContext));
handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, NULL, NULL); handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, NULL);
} }
tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, TD_VID(pTq->pVnode)); tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, TD_VID(pTq->pVnode));
taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle));
} }
end:
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
tdbTbcClose(pCur); tdbTbcClose(pCur);
return 0; return code;
} }
source/dnode/vnode/src/tq/tqRead.c
浏览文件 @ 6811a6b2
...@@ -183,23 +183,24 @@ end: ...@@ -183,23 +183,24 @@ end:
return tbSuid == realTbSuid; return tbSuid == realTbSuid;
} }
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) { int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead, uint64_t reqId) {
int32_t code = 0; int32_t code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
taosThreadMutexLock(&pHandle->pWalReader->mutex); taosThreadMutexLock(&pHandle->pWalReader->mutex);
int64_t offset = *fetchOffset; int64_t offset = *fetchOffset;
while (1) { while (1) {
if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) { if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return", tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return, reqId:0x%"PRIx64,
pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset); pHandle->consumerId, pHandle->epoch, vgId, offset, reqId);
*fetchOffset = offset - 1; *fetchOffset = offset - 1;
code = -1; code = -1;
goto END; goto END;
} }
tqDebug("vgId:%d, taosx get msg ver %" PRId64 ", type: %s", pTq->pVnode->config.vgId, offset, tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64, vgId,
TMSG_INFO((*ppCkHead)->head.msgType)); pHandle->consumerId, offset, TMSG_INFO((*ppCkHead)->head.msgType), reqId);
if ((*ppCkHead)->head.msgType == TDMT_VND_SUBMIT) { if ((*ppCkHead)->head.msgType == TDMT_VND_SUBMIT) {
code = walFetchBody(pHandle->pWalReader, ppCkHead); code = walFetchBody(pHandle->pWalReader, ppCkHead);
......
source/dnode/vnode/src/tq/tqSink.c
浏览文件 @ 6811a6b2
...@@ -370,11 +370,6 @@ int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { ...@@ -370,11 +370,6 @@ int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_error:
terrno = TSDB_CODE_OUT_OF_MEMORY;
tqError("failed to encode submit req since %s", terrstr());
return TSDB_CODE_OUT_OF_MEMORY;
} }
void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
...@@ -441,9 +436,6 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -441,9 +436,6 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
for (int32_t rowId = 0; rowId < rows; rowId++) { for (int32_t rowId = 0; rowId < rows; rowId++) {
SVCreateTbReq createTbReq = {0}; SVCreateTbReq createTbReq = {0};
SVCreateTbReq* pCreateTbReq = &createTbReq; SVCreateTbReq* pCreateTbReq = &createTbReq;
if (!pCreateTbReq) {
goto _end;
}
// set const // set const
pCreateTbReq->flags = 0; pCreateTbReq->flags = 0;
...@@ -460,6 +452,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -460,6 +452,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
if (size == 2) { if (size == 2) {
tagArray = taosArrayInit(1, sizeof(STagVal)); tagArray = taosArrayInit(1, sizeof(STagVal));
if (!tagArray) { if (!tagArray) {
tdDestroySVCreateTbReq(pCreateTbReq);
goto _end; goto _end;
} }
STagVal tagVal = { STagVal tagVal = {
...@@ -477,6 +470,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -477,6 +470,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
} else { } else {
tagArray = taosArrayInit(size - 1, sizeof(STagVal)); tagArray = taosArrayInit(size - 1, sizeof(STagVal));
if (!tagArray) { if (!tagArray) {
tdDestroySVCreateTbReq(pCreateTbReq);
goto _end; goto _end;
} }
for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
...@@ -503,6 +497,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -503,6 +497,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
tTagNew(tagArray, 1, false, &pTag); tTagNew(tagArray, 1, false, &pTag);
tagArray = taosArrayDestroy(tagArray); tagArray = taosArrayDestroy(tagArray);
if (pTag == NULL) { if (pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
} }
...@@ -556,6 +551,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -556,6 +551,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
SVCreateTbReq* pCreateTbReq = NULL; SVCreateTbReq* pCreateTbReq = NULL;
if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) {
taosMemoryFree(ctbName);
goto _end; goto _end;
}; };
...@@ -572,6 +568,8 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -572,6 +568,8 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
// set tag content // set tag content
tagArray = taosArrayInit(1, sizeof(STagVal)); tagArray = taosArrayInit(1, sizeof(STagVal));
if (!tagArray) { if (!tagArray) {
taosMemoryFree(ctbName);
tdDestroySVCreateTbReq(pCreateTbReq);
goto _end; goto _end;
} }
STagVal tagVal = { STagVal tagVal = {
...@@ -586,6 +584,8 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -586,6 +584,8 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
tTagNew(tagArray, 1, false, &pTag); tTagNew(tagArray, 1, false, &pTag);
tagArray = taosArrayDestroy(tagArray); tagArray = taosArrayDestroy(tagArray);
if (pTag == NULL) { if (pTag == NULL) {
taosMemoryFree(ctbName);
tdDestroySVCreateTbReq(pCreateTbReq);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
} }
...@@ -630,6 +630,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -630,6 +630,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
// rows // rows
if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) {
taosArrayDestroy(tbData.aRowP); taosArrayDestroy(tbData.aRowP);
tdDestroySVCreateTbReq(tbData.pCreateTbReq);
goto _end; goto _end;
} }
...@@ -680,6 +681,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -680,6 +681,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
SSubmitReq2 submitReq = {0}; SSubmitReq2 submitReq = {0};
if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) {
tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end; goto _end;
} }
...@@ -693,6 +695,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -693,6 +695,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
len += sizeof(SSubmitReq2Msg); len += sizeof(SSubmitReq2Msg);
pBuf = rpcMallocCont(len); pBuf = rpcMallocCont(len);
if (NULL == pBuf) { if (NULL == pBuf) {
tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE);
goto _end; goto _end;
} }
((SSubmitReq2Msg*)pBuf)->header.vgId = TD_VID(pVnode); ((SSubmitReq2Msg*)pBuf)->header.vgId = TD_VID(pVnode);
...@@ -704,6 +707,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* ...@@ -704,6 +707,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
tqError("failed to encode submit req since %s", terrstr()); tqError("failed to encode submit req since %s", terrstr());
tEncoderClear(&encoder); tEncoderClear(&encoder);
rpcFreeCont(pBuf); rpcFreeCont(pBuf);
tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE);
continue; continue;
} }
tEncoderClear(&encoder); tEncoderClear(&encoder);
......
source/dnode/vnode/src/tsdb/tsdbCache.c
浏览文件 @ 6811a6b2
...@@ -658,6 +658,9 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa ...@@ -658,6 +658,9 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
bool hasVal = false; bool hasVal = false;
state->row = tMergeTreeGetRow(&state->mergeTree); state->row = tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row; *ppRow = &state->row;
if (nCols != state->pLoadInfo->numOfCols) {
state->pLoadInfo->numOfCols = nCols;
}
hasVal = tMergeTreeNext(&state->mergeTree); hasVal = tMergeTreeNext(&state->mergeTree);
if (TSDBROW_TS(&state->row) <= state->lastTs) { if (TSDBROW_TS(&state->row) <= state->lastTs) {
*pIgnoreEarlierTs = true; *pIgnoreEarlierTs = true;
......
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
浏览文件 @ 6811a6b2
...@@ -134,7 +134,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id ...@@ -134,7 +134,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id
// all queried tables have been dropped already, return immediately. // all queried tables have been dropped already, return immediately.
if (p->pSchema == NULL) { if (p->pSchema == NULL) {
taosMemoryFree(p);
tsdbWarn("all queried tables has been dropped, try next group, %s", idstr); tsdbWarn("all queried tables has been dropped, try next group, %s", idstr);
return TSDB_CODE_PAR_TABLE_NOT_EXIST; return TSDB_CODE_PAR_TABLE_NOT_EXIST;
} }
...@@ -332,6 +331,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 ...@@ -332,6 +331,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
// retrieve the only one last row of all tables in the uid list. // retrieve the only one last row of all tables in the uid list.
if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) { if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
int64_t totalLastTs = INT64_MAX;
for (int32_t i = 0; i < pr->numOfTables; ++i) { for (int32_t i = 0; i < pr->numOfTables; ++i) {
STableKeyInfo* pKeyInfo = &pr->pTableList[i]; STableKeyInfo* pKeyInfo = &pr->pTableList[i];
...@@ -350,7 +350,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 ...@@ -350,7 +350,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
{ {
bool hasNotNullRow = true; bool hasNotNullRow = true;
int64_t minTs = INT64_MAX; int64_t singleTableLastTs = INT64_MAX;
for (int32_t k = 0; k < pr->numOfCols; ++k) { for (int32_t k = 0; k < pr->numOfCols; ++k) {
int32_t slotId = slotIds[k]; int32_t slotId = slotIds[k];
...@@ -361,7 +361,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 ...@@ -361,7 +361,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
hasRes = true; hasRes = true;
p->ts = pCol->ts; p->ts = pCol->ts;
p->colVal = pCol->colVal; p->colVal = pCol->colVal;
minTs = pCol->ts; singleTableLastTs = pCol->ts;
// only set value for last row query // only set value for last row query
if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST_ROW)) { if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST_ROW)) {
...@@ -386,8 +386,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 ...@@ -386,8 +386,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
hasRes = true; hasRes = true;
p->ts = pColVal->ts; p->ts = pColVal->ts;
if (pColVal->ts < minTs && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { if (pColVal->ts < singleTableLastTs && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) {
minTs = pColVal->ts; singleTableLastTs = pColVal->ts;
} }
if (!IS_VAR_DATA_TYPE(pColVal->colVal.type)) { if (!IS_VAR_DATA_TYPE(pColVal->colVal.type)) {
...@@ -407,9 +407,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 ...@@ -407,9 +407,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
} }
if (hasNotNullRow) { if (hasNotNullRow) {
if (INT64_MAX == totalLastTs || (INT64_MAX != singleTableLastTs && totalLastTs < singleTableLastTs)) {
totalLastTs = singleTableLastTs;
}
double cost = (taosGetTimestampUs() - st) / 1000.0; double cost = (taosGetTimestampUs() - st) / 1000.0;
if (cost > tsCacheLazyLoadThreshold) { if (cost > tsCacheLazyLoadThreshold) {
pr->lastTs = minTs; pr->lastTs = totalLastTs;
} }
} }
} }
......
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ 6811a6b2
...@@ -427,7 +427,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf ...@@ -427,7 +427,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf
return pTableMap; return pTableMap;
} }
static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts, int32_t step) {
STableBlockScanInfo** p = NULL; STableBlockScanInfo** p = NULL;
while ((p = taosHashIterate(pTableMap, p)) != NULL) { while ((p = taosHashIterate(pTableMap, p)) != NULL) {
STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p;
...@@ -446,6 +446,7 @@ static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { ...@@ -446,6 +446,7 @@ static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) {
pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline);
pInfo->lastKey = ts; pInfo->lastKey = ts;
pInfo->lastKeyInStt = ts + step;
} }
} }
...@@ -921,7 +922,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN ...@@ -921,7 +922,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
pBlockNum->numOfBlocks += 1; pBlockNum->numOfBlocks += 1;
} }
if ((pScanInfo->pBlockList != NULL) && (taosArrayGetSize(pScanInfo->pBlockList) > 0)) { if (taosArrayGetSize(pScanInfo->pBlockList) > 0) {
numOfQTable += 1; numOfQTable += 1;
} }
} }
...@@ -2471,7 +2472,6 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan ...@@ -2471,7 +2472,6 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
initMemDataIterator(pScanInfo, pReader); initMemDataIterator(pScanInfo, pReader);
pLBlockReader->uid = pScanInfo->uid; pLBlockReader->uid = pScanInfo->uid;
int32_t step = ASCENDING_TRAVERSE(pLBlockReader->order) ? 1 : -1;
STimeWindow w = pLBlockReader->window; STimeWindow w = pLBlockReader->window;
if (ASCENDING_TRAVERSE(pLBlockReader->order)) { if (ASCENDING_TRAVERSE(pLBlockReader->order)) {
w.skey = pScanInfo->lastKeyInStt; w.skey = pScanInfo->lastKeyInStt;
...@@ -4220,12 +4220,8 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { ...@@ -4220,12 +4220,8 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) {
if (pStatus->loadFromFile) { if (pStatus->loadFromFile) {
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
if (pBlockInfo != NULL) { if (pBlockInfo != NULL) {
pBlockScanInfo = pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr);
*(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
if (pBlockScanInfo == NULL) { if (pBlockScanInfo == NULL) {
code = TSDB_CODE_INVALID_PARA;
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid,
taosHashGetSize(pReader->status.pTableMap), pReader->idStr);
goto _err; goto _err;
} }
} else { } else {
...@@ -4457,8 +4453,9 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { ...@@ -4457,8 +4453,9 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
if (pReader->step == EXTERNAL_ROWS_PREV) { if (pReader->step == EXTERNAL_ROWS_PREV) {
// prepare for the main scan // prepare for the main scan
int32_t code = doOpenReaderImpl(pReader); code = doOpenReaderImpl(pReader);
resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->window.ekey); int32_t step = 1;
resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->window.ekey, step);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
...@@ -4479,8 +4476,9 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { ...@@ -4479,8 +4476,9 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) {
// prepare for the next row scan // prepare for the next row scan
int32_t code = doOpenReaderImpl(pReader->innerReader[1]); int32_t step = -1;
resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey); code = doOpenReaderImpl(pReader->innerReader[1]);
resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey, step);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
...@@ -4683,15 +4681,12 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { ...@@ -4683,15 +4681,12 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
} }
if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) {
tsdbDebug("tsdb reader reset return %p", pReader->pReadSnap); tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr);
tsdbReleaseReader(pReader); tsdbReleaseReader(pReader);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
SDataBlockIter* pBlockIter = &pStatus->blockIter; SDataBlockIter* pBlockIter = &pStatus->blockIter;
pReader->order = pCond->order; pReader->order = pCond->order;
...@@ -4712,8 +4707,10 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { ...@@ -4712,8 +4707,10 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
resetDataBlockIterator(pBlockIter, pReader->order); resetDataBlockIterator(pBlockIter, pReader->order);
resetTableListIndex(&pReader->status); resetTableListIndex(&pReader->status);
int64_t ts = ASCENDING_TRAVERSE(pReader->order) ? pReader->window.skey - 1 : pReader->window.ekey + 1; bool asc = ASCENDING_TRAVERSE(pReader->order);
resetAllDataBlockScanInfo(pStatus->pTableMap, ts); int32_t step = asc? 1:-1;
int64_t ts = asc? pReader->window.skey - 1 : pReader->window.ekey + 1;
resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step);
int32_t code = 0; int32_t code = 0;
...@@ -4728,7 +4725,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { ...@@ -4728,7 +4725,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr); numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr);
tsdbReleaseReader(pReader); tsdbReleaseReader(pReader);
return code; return code;
} }
} }
...@@ -4995,3 +4991,9 @@ void tsdbUntakeReadSnap(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proacti ...@@ -4995,3 +4991,9 @@ void tsdbUntakeReadSnap(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proacti
} }
tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode));
} }
// if failed, do nothing
void tsdbReaderSetId(STsdbReader* pReader, const char* idstr) {
taosMemoryFreeClear(pReader->idStr);
pReader->idStr = taosStrdup(idstr);
}
\ No newline at end of file
source/libs/executor/inc/executorimpl.h
浏览文件 @ 6811a6b2
...@@ -115,6 +115,7 @@ typedef struct STaskIdInfo { ...@@ -115,6 +115,7 @@ typedef struct STaskIdInfo {
uint64_t subplanId; uint64_t subplanId;
uint64_t templateId; uint64_t templateId;
char* str; char* str;
int32_t vgId;
} STaskIdInfo; } STaskIdInfo;
enum { enum {
...@@ -834,8 +835,10 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); ...@@ -834,8 +835,10 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode);
void doDestroyTask(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo);
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, char* buildTaskId(uint64_t taskId, uint64_t queryId);
char* sql, EOPTR_EXEC_MODEL model);
int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
int32_t vgId, char* sql, EOPTR_EXEC_MODEL model);
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
......
source/libs/executor/inc/tsort.h
浏览文件 @ 6811a6b2
...@@ -44,7 +44,8 @@ typedef struct SSortSource { ...@@ -44,7 +44,8 @@ typedef struct SSortSource {
void* param; void* param;
bool onlyRef; bool onlyRef;
}; };
int64_t fetchUs;
int64_t fetchNum;
} SSortSource; } SSortSource;
typedef struct SMsortComparParam { typedef struct SMsortComparParam {
......
source/libs/executor/src/exchangeoperator.c
浏览文件 @ 6811a6b2
...@@ -512,8 +512,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo ...@@ -512,8 +512,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
} }
// todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator.
blockDataUpdateTsWindow(pRes, 0);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
source/libs/executor/src/executil.c
浏览文件 @ 6811a6b2
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
#include "functionMgt.h" #include "functionMgt.h"
#include "index.h" #include "index.h"
#include "os.h" #include "os.h"
#include "query.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "thash.h" #include "thash.h"
#include "tmsg.h" #include "tmsg.h"
...@@ -1058,10 +1059,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, ...@@ -1058,10 +1059,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
SIndexMetaArg metaArg = { SIndexMetaArg metaArg = {
.metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid};
SIdxFltStatus status = SFLT_NOT_INDEX; status = SFLT_NOT_INDEX;
code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status);
if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake
// qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid);
code = TDB_CODE_SUCCESS; code = TDB_CODE_SUCCESS;
} else { } else {
qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList));
......
source/libs/executor/src/executor.c
浏览文件 @ 6811a6b2
...@@ -159,6 +159,30 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu ...@@ -159,6 +159,30 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
} }
} }
void doSetTaskId(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamScanInfo* pStreamScanInfo = pOperator->info;
STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info;
if (pScanInfo->base.dataReader != NULL) {
tsdbReaderSetId(pScanInfo->base.dataReader, pTaskInfo->id.str);
}
} else {
doSetTaskId(pOperator->pDownstream[0]);
}
}
void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) {
SExecTaskInfo* pTaskInfo = tinfo;
pTaskInfo->id.queryId = queryId;
taosMemoryFreeClear(pTaskInfo->id.str);
pTaskInfo->id.str = buildTaskId(taskId, queryId);
// set the idstr for tsdbReader
doSetTaskId(pTaskInfo->pRoot);
}
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
if (tinfo == NULL) { if (tinfo == NULL) {
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
...@@ -218,7 +242,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, ...@@ -218,7 +242,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
return code; return code;
} }
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) { qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, SSchemaWrapper** pSchema) {
if (msg == NULL) { if (msg == NULL) {
// create raw scan // create raw scan
SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
...@@ -231,7 +255,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n ...@@ -231,7 +255,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n
pTaskInfo->cost.created = taosGetTimestampUs(); pTaskInfo->cost.created = taosGetTimestampUs();
pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE; pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE;
pTaskInfo->pRoot = createRawScanOperatorInfo(readers, pTaskInfo); pTaskInfo->pRoot = createRawScanOperatorInfo(pReaderHandle, pTaskInfo);
if (NULL == pTaskInfo->pRoot) { if (NULL == pTaskInfo->pRoot) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pTaskInfo); taosMemoryFree(pTaskInfo);
...@@ -248,7 +272,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n ...@@ -248,7 +272,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n
} }
qTaskInfo_t pTaskInfo = NULL; qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(readers, 0, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_QUEUE); code = qCreateExecTask(pReaderHandle, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_QUEUE);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
nodesDestroyNode((SNode*)pPlan); nodesDestroyNode((SNode*)pPlan);
qDestroyTask(pTaskInfo); qDestroyTask(pTaskInfo);
...@@ -274,13 +298,11 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n ...@@ -274,13 +298,11 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n
return pTaskInfo; return pTaskInfo;
} }
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId) {
if (msg == NULL) { if (msg == NULL) {
return NULL; return NULL;
} }
/*qDebugL("stream task string %s", (const char*)msg);*/
struct SSubplan* pPlan = NULL; struct SSubplan* pPlan = NULL;
int32_t code = qStringToSubplan(msg, &pPlan); int32_t code = qStringToSubplan(msg, &pPlan);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
...@@ -289,7 +311,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { ...@@ -289,7 +311,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) {
} }
qTaskInfo_t pTaskInfo = NULL; qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(readers, 0, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); code = qCreateExecTask(readers, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
nodesDestroyNode((SNode*)pPlan); nodesDestroyNode((SNode*)pPlan);
qDestroyTask(pTaskInfo); qDestroyTask(pTaskInfo);
...@@ -468,11 +490,11 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, ...@@ -468,11 +490,11 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
taosThreadOnce(&initPoolOnce, initRefPool); taosThreadOnce(&initPoolOnce, initRefPool);
qDebug("start to create subplan task, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); qDebug("start to create task, TID:0x%" PRIx64 " QID:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId);
int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); int32_t code = createExecTaskInfo(pSubplan, pTask, readHandle, taskId, vgId, sql, model);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to createExecTaskInfoImpl, code: %s", tstrerror(code)); qError("failed to createExecTaskInfo, code: %s", tstrerror(code));
goto _error; goto _error;
} }
...@@ -1101,7 +1123,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1101,7 +1123,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList);
qDebug("switch to next table %" PRId64 " ts %" PRId64 "% "PRId64 " rows returned", uid, ts, pInfo->pTableScanOp->resultInfo.totalRows); qDebug("switch to table uid:%" PRId64 " ts:%" PRId64 "% "PRId64 " rows returned", uid, ts, pInfo->pTableScanOp->resultInfo.totalRows);
pInfo->pTableScanOp->resultInfo.totalRows = 0; pInfo->pTableScanOp->resultInfo.totalRows = 0;
bool found = false; bool found = false;
...@@ -1140,7 +1162,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ...@@ -1140,7 +1162,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pTableScanInfo->base.cond.twindows.skey = oldSkey; pTableScanInfo->base.cond.twindows.skey = oldSkey;
pTableScanInfo->scanTimes = 0; pTableScanInfo->scanTimes = 0;
qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid,
ts, pTableScanInfo->currentTable, numOfTables); ts, pTableScanInfo->currentTable, numOfTables);
} else { } else {
qError("invalid pOffset->type:%d", pOffset->type); qError("invalid pOffset->type:%d", pOffset->type);
......
source/libs/executor/src/executorimpl.c
浏览文件 @ 6811a6b2
...@@ -1959,7 +1959,7 @@ void destroyAggOperatorInfo(void* param) { ...@@ -1959,7 +1959,7 @@ void destroyAggOperatorInfo(void* param) {
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
static char* buildTaskId(uint64_t taskId, uint64_t queryId) { char* buildTaskId(uint64_t taskId, uint64_t queryId) {
char* p = taosMemoryMalloc(64); char* p = taosMemoryMalloc(64);
int32_t offset = 6; int32_t offset = 6;
...@@ -1971,11 +1971,10 @@ static char* buildTaskId(uint64_t taskId, uint64_t queryId) { ...@@ -1971,11 +1971,10 @@ static char* buildTaskId(uint64_t taskId, uint64_t queryId) {
offset += tintToHex(queryId, &p[offset]); offset += tintToHex(queryId, &p[offset]);
p[offset] = 0; p[offset] = 0;
return p; return p;
} }
static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model, char* dbFName) { static SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, char* dbFName) {
SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
if (pTaskInfo == NULL) { if (pTaskInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
...@@ -1990,6 +1989,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT ...@@ -1990,6 +1989,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo));
pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES);
pTaskInfo->id.vgId = vgId;
pTaskInfo->id.queryId = queryId; pTaskInfo->id.queryId = queryId;
pTaskInfo->id.str = buildTaskId(taskId, queryId); pTaskInfo->id.str = buildTaskId(taskId, queryId);
return pTaskInfo; return pTaskInfo;
...@@ -2178,7 +2178,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo ...@@ -2178,7 +2178,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
#ifndef NDEBUG #ifndef NDEBUG
int32_t sz = tableListGetSize(pTableListInfo); int32_t sz = tableListGetSize(pTableListInfo);
qDebug("create stream task, total:%d", sz); qDebug("vgId:%d create stream task, total qualified tables:%d, %s", pTaskInfo->id.vgId, sz, idstr);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i);
...@@ -2439,17 +2439,14 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT ...@@ -2439,17 +2439,14 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
char* sql, EOPTR_EXEC_MODEL model) { int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) {
uint64_t queryId = pPlan->id.queryId; *pTaskInfo = doCreateExecTaskInfo(pPlan->id.queryId, taskId, vgId, model, pPlan->dbFName);
*pTaskInfo = createExecTaskInfo(queryId, taskId, model, pPlan->dbFName);
if (*pTaskInfo == NULL) { if (*pTaskInfo == NULL) {
goto _complete; goto _complete;
} }
if (pHandle) { if (pHandle) {
/*(*pTaskInfo)->streamInfo.fillHistoryVer1 = pHandle->fillHistoryVer1;*/
if (pHandle->pStateBackend) { if (pHandle->pStateBackend) {
(*pTaskInfo)->streamInfo.pState = pHandle->pStateBackend; (*pTaskInfo)->streamInfo.pState = pHandle->pStateBackend;
} }
......
source/libs/executor/src/scanoperator.c
浏览文件 @ 6811a6b2
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include "thash.h" #include "thash.h"
#include "ttypes.h" #include "ttypes.h"
#define MULTI_READER_MAX_TABLE_NUM 5000
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
...@@ -43,7 +44,9 @@ typedef struct STableMergeScanSortSourceParam { ...@@ -43,7 +44,9 @@ typedef struct STableMergeScanSortSourceParam {
SOperatorInfo* pOperator; SOperatorInfo* pOperator;
int32_t readerIdx; int32_t readerIdx;
uint64_t uid; uint64_t uid;
SSDataBlock* inputBlock; SSDataBlock* inputBlock;
bool multiReader;
STsdbReader* dataReader;
} STableMergeScanSortSourceParam; } STableMergeScanSortSourceParam;
static bool processBlockWithProbability(const SSampleExecInfo* pInfo); static bool processBlockWithProbability(const SSampleExecInfo* pInfo);
...@@ -1194,6 +1197,8 @@ static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, ...@@ -1194,6 +1197,8 @@ static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pKey); SET_SESSION_WIN_KEY_INVALID(pKey);
} }
taosMemoryFree(pCur);
return code; return code;
} }
...@@ -2586,6 +2591,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { ...@@ -2586,6 +2591,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t readIdx = source->readerIdx; int32_t readIdx = source->readerIdx;
SSDataBlock* pBlock = source->inputBlock; SSDataBlock* pBlock = source->inputBlock;
int32_t code = 0;
SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx);
...@@ -2593,17 +2599,20 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { ...@@ -2593,17 +2599,20 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
SReadHandle* pHandle = &pInfo->base.readHandle; SReadHandle* pHandle = &pInfo->base.readHandle;
int32_t code = if (NULL == source->dataReader || !source->multiReader) {
tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo));
if (code != 0) { if (code != 0) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
}
} }
pInfo->base.dataReader = source->dataReader;
STsdbReader* reader = pInfo->base.dataReader; STsdbReader* reader = pInfo->base.dataReader;
qTrace("tsdb/read-table-data: %p, enter next reader", reader); qTrace("tsdb/read-table-data: %p, enter next reader", reader);
while (tsdbNextDataBlock(reader)) { while (tsdbNextDataBlock(reader)) {
if (isTaskKilled(pTaskInfo)) { if (isTaskKilled(pTaskInfo)) {
tsdbReleaseDataBlock(reader); tsdbReleaseDataBlock(reader);
pInfo->base.dataReader = NULL;
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
} }
...@@ -2637,14 +2646,18 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { ...@@ -2637,14 +2646,18 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
qTrace("tsdb/read-table-data: %p, close reader", reader); qTrace("tsdb/read-table-data: %p, close reader", reader);
tsdbReaderClose(pInfo->base.dataReader); if (!source->multiReader) {
tsdbReaderClose(pInfo->base.dataReader);
source->dataReader = NULL;
}
pInfo->base.dataReader = NULL; pInfo->base.dataReader = NULL;
return pBlock; return pBlock;
} }
qDebug("8"); if (!source->multiReader) {
tsdbReaderClose(pInfo->base.dataReader);
tsdbReaderClose(pInfo->base.dataReader); source->dataReader = NULL;
}
pInfo->base.dataReader = NULL; pInfo->base.dataReader = NULL;
return NULL; return NULL;
} }
...@@ -2716,6 +2729,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { ...@@ -2716,6 +2729,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanSortSourceParam param = {0}; STableMergeScanSortSourceParam param = {0};
param.readerIdx = i; param.readerIdx = i;
param.pOperator = pOperator; param.pOperator = pOperator;
param.multiReader = (numOfTable <= MULTI_READER_MAX_TABLE_NUM) ? true : false;
param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity); blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity);
...@@ -2759,6 +2773,8 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { ...@@ -2759,6 +2773,8 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
for (int32_t i = 0; i < numOfTable; ++i) { for (int32_t i = 0; i < numOfTable; ++i) {
STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
blockDataDestroy(param->inputBlock); blockDataDestroy(param->inputBlock);
tsdbReaderClose(param->dataReader);
param->dataReader = NULL;
} }
taosArrayClear(pInfo->sortSourceParams); taosArrayClear(pInfo->sortSourceParams);
...@@ -2801,9 +2817,6 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* ...@@ -2801,9 +2817,6 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock*
qDebug("%s get sorted row block, rows:%d, limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, qDebug("%s get sorted row block, rows:%d, limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows,
pInfo->limitInfo.numOfOutputRows); pInfo->limitInfo.numOfOutputRows);
if (limitReached) {
resetLimitInfoForNextGroup(&pInfo->limitInfo);
}
return (pResBlock->info.rows > 0) ? pResBlock : NULL; return (pResBlock->info.rows > 0) ? pResBlock : NULL;
} }
...@@ -2856,6 +2869,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { ...@@ -2856,6 +2869,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
pInfo->tableStartIndex = pInfo->tableEndIndex + 1; pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
pInfo->groupId = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex)->groupId; pInfo->groupId = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex)->groupId;
startGroupTableMergeScan(pOperator); startGroupTableMergeScan(pOperator);
resetLimitInfoForNextGroup(&pInfo->limitInfo);
} }
} }
...@@ -2871,15 +2885,17 @@ void destroyTableMergeScanOperatorInfo(void* param) { ...@@ -2871,15 +2885,17 @@ void destroyTableMergeScanOperatorInfo(void* param) {
for (int32_t i = 0; i < numOfTable; i++) { for (int32_t i = 0; i < numOfTable; i++) {
STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i);
blockDataDestroy(p->inputBlock); blockDataDestroy(p->inputBlock);
tsdbReaderClose(p->dataReader);
p->dataReader = NULL;
} }
tsdbReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL;
taosArrayDestroy(pTableScanInfo->sortSourceParams); taosArrayDestroy(pTableScanInfo->sortSourceParams);
tsortDestroySortHandle(pTableScanInfo->pSortHandle); tsortDestroySortHandle(pTableScanInfo->pSortHandle);
pTableScanInfo->pSortHandle = NULL; pTableScanInfo->pSortHandle = NULL;
tsdbReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL;
for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) { for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) {
SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i); SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i);
taosMemoryFree(pCond->colList); taosMemoryFree(pCond->colList);
...@@ -2896,8 +2912,6 @@ void destroyTableMergeScanOperatorInfo(void* param) { ...@@ -2896,8 +2912,6 @@ void destroyTableMergeScanOperatorInfo(void* param) {
taosArrayDestroy(pTableScanInfo->pSortInfo); taosArrayDestroy(pTableScanInfo->pSortInfo);
cleanupExprSupp(&pTableScanInfo->base.pseudoSup); cleanupExprSupp(&pTableScanInfo->base.pseudoSup);
tsdbReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL;
taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache); taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
......
source/libs/executor/src/sysscanoperator.c
浏览文件 @ 6811a6b2
...@@ -57,9 +57,11 @@ typedef struct SSysTableScanInfo { ...@@ -57,9 +57,11 @@ typedef struct SSysTableScanInfo {
const char* pUser; const char* pUser;
bool sysInfo; bool sysInfo;
bool showRewrite; bool showRewrite;
bool restore;
SNode* pCondition; // db_name filter condition, to discard data that are not in current database SNode* pCondition; // db_name filter condition, to discard data that are not in current database
SMTbCursor* pCur; // cursor for iterate the local table meta store. SMTbCursor* pCur; // cursor for iterate the local table meta store.
SSysTableIndex* pIdx; // idx for local table meta SSysTableIndex* pIdx; // idx for local table meta
SHashObj* pSchema;
SColMatchInfo matchInfo; SColMatchInfo matchInfo;
SName name; SName name;
SSDataBlock* pRes; SSDataBlock* pRes;
...@@ -514,9 +516,23 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { ...@@ -514,9 +516,23 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
} }
SHashObj* stableSchema = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (pInfo->pSchema == NULL) {
taosHashSetFreeFp(stableSchema, tDeleteSSchemaWrapperForHash); pInfo->pSchema = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0) { taosHashSetFreeFp(pInfo->pSchema, tDeleteSSchemaWrapperForHash);
}
if (!pInfo->pCur || !pInfo->pSchema) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
qError("sysTableScanUserCols failed since %s", terrstr(terrno));
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
int32_t restore = pInfo->restore;
pInfo->restore = false;
while (restore || ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) {
if (restore) restore = false;
char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
...@@ -524,33 +540,36 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { ...@@ -524,33 +540,36 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
if (pInfo->pCur->mr.me.type == TSDB_SUPER_TABLE) { if (pInfo->pCur->mr.me.type == TSDB_SUPER_TABLE) {
qDebug("sysTableScanUserCols cursor get super table"); qDebug("sysTableScanUserCols cursor get super table");
void* schema = taosHashGet(stableSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t)); void* schema = taosHashGet(pInfo->pSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t));
if (schema == NULL) { if (schema == NULL) {
SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&pInfo->pCur->mr.me.stbEntry.schemaRow); SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&pInfo->pCur->mr.me.stbEntry.schemaRow);
taosHashPut(stableSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); taosHashPut(pInfo->pSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES);
} }
continue; continue;
} else if (pInfo->pCur->mr.me.type == TSDB_CHILD_TABLE) { } else if (pInfo->pCur->mr.me.type == TSDB_CHILD_TABLE) {
qDebug("sysTableScanUserCols cursor get child table"); qDebug("sysTableScanUserCols cursor get child table");
STR_TO_VARSTR(typeName, "CHILD_TABLE"); STR_TO_VARSTR(typeName, "CHILD_TABLE");
STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name);
int64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
void* schema = taosHashGet(stableSchema, &pInfo->pCur->mr.me.ctbEntry.suid, sizeof(int64_t)); void* schema = taosHashGet(pInfo->pSchema, &pInfo->pCur->mr.me.ctbEntry.suid, sizeof(int64_t));
if (schema != NULL) { if (schema != NULL) {
schemaRow = *(SSchemaWrapper**)schema; schemaRow = *(SSchemaWrapper**)schema;
} else { } else {
tDecoderClear(&pInfo->pCur->mr.coder); SMetaReader smrSuperTable = {0};
int code = metaGetTableEntryByUid(&pInfo->pCur->mr, suid); metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
int code = metaGetTableEntryByUid(&smrSuperTable, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByName, therefore, return directly // terrno has been set by metaGetTableEntryByName, therefore, return directly
qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code);
metaReaderClear(&smrSuperTable);
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0; pInfo->loadInfo.totalRows = 0;
taosHashCleanup(stableSchema);
return NULL; return NULL;
} }
schemaRow = &pInfo->pCur->mr.me.stbEntry.schemaRow; SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow);
taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES);
schemaRow = schemaWrapper;
metaReaderClear(&smrSuperTable);
} }
} else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) {
qDebug("sysTableScanUserCols cursor get normal table"); qDebug("sysTableScanUserCols cursor get normal table");
...@@ -565,8 +584,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { ...@@ -565,8 +584,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) { if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
numOfRows = 0; numOfRows = 0;
pInfo->restore = true;
metaTbCursorPrev(pInfo->pCur);
if (pInfo->pRes->info.rows > 0) { if (pInfo->pRes->info.rows > 0) {
break; break;
...@@ -576,8 +594,6 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { ...@@ -576,8 +594,6 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
} }
} }
taosHashCleanup(stableSchema);
if (numOfRows > 0) { if (numOfRows > 0) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
numOfRows = 0; numOfRows = 0;
...@@ -697,7 +713,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { ...@@ -697,7 +713,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
} }
if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) { if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) {
metaTbCursorPrev(pInfo->pCur); metaTbCursorPrev(pInfo->pCur, TSDB_TABLE_MAX);
blockFull = true; blockFull = true;
} else { } else {
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows,
...@@ -1791,6 +1807,11 @@ void destroySysScanOperator(void* param) { ...@@ -1791,6 +1807,11 @@ void destroySysScanOperator(void* param) {
pInfo->pIdx = NULL; pInfo->pIdx = NULL;
} }
if(pInfo->pSchema) {
taosHashCleanup(pInfo->pSchema);
pInfo->pSchema = NULL;
}
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);
taosMemoryFreeClear(pInfo->pUser); taosMemoryFreeClear(pInfo->pUser);
......
source/libs/executor/src/tsort.c
浏览文件 @ 6811a6b2
...@@ -108,12 +108,18 @@ static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { ...@@ -108,12 +108,18 @@ static int32_t sortComparCleanup(SMsortComparParam* cmpParam) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void tsortClearOrderdSource(SArray* pOrderedSource) { void tsortClearOrderdSource(SArray* pOrderedSource, int64_t *fetchUs, int64_t *fetchNum) {
for (size_t i = 0; i < taosArrayGetSize(pOrderedSource); i++) { for (size_t i = 0; i < taosArrayGetSize(pOrderedSource); i++) {
SSortSource** pSource = taosArrayGet(pOrderedSource, i); SSortSource** pSource = taosArrayGet(pOrderedSource, i);
if (NULL == *pSource) { if (NULL == *pSource) {
continue; continue;
} }
if (fetchUs) {
*fetchUs += (*pSource)->fetchUs;
*fetchNum += (*pSource)->fetchNum;
}
// release pageIdList // release pageIdList
if ((*pSource)->pageIdList) { if ((*pSource)->pageIdList) {
taosArrayDestroy((*pSource)->pageIdList); taosArrayDestroy((*pSource)->pageIdList);
...@@ -147,7 +153,10 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { ...@@ -147,7 +153,10 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) {
taosMemoryFreeClear(pSortHandle->idStr); taosMemoryFreeClear(pSortHandle->idStr);
blockDataDestroy(pSortHandle->pDataBlock); blockDataDestroy(pSortHandle->pDataBlock);
tsortClearOrderdSource(pSortHandle->pOrderedSource); int64_t fetchUs = 0, fetchNum = 0;
tsortClearOrderdSource(pSortHandle->pOrderedSource, &fetchUs, &fetchNum);
qError("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr);
taosArrayDestroy(pSortHandle->pOrderedSource); taosArrayDestroy(pSortHandle->pOrderedSource);
taosMemoryFreeClear(pSortHandle); taosMemoryFreeClear(pSortHandle);
} }
...@@ -307,7 +316,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 ...@@ -307,7 +316,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32
} }
int64_t et = taosGetTimestampUs(); int64_t et = taosGetTimestampUs();
qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); qError("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr);
} }
return code; return code;
...@@ -365,7 +374,10 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT ...@@ -365,7 +374,10 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
releaseBufPage(pHandle->pBuf, pPage); releaseBufPage(pHandle->pBuf, pPage);
} }
} else { } else {
int64_t st = taosGetTimestampUs();
pSource->src.pBlock = pHandle->fetchfp(((SSortSource*)pSource)->param); pSource->src.pBlock = pHandle->fetchfp(((SSortSource*)pSource)->param);
pSource->fetchUs += taosGetTimestampUs() - st;
pSource->fetchNum++;
if (pSource->src.pBlock == NULL) { if (pSource->src.pBlock == NULL) {
(*numOfCompleted) += 1; (*numOfCompleted) += 1;
pSource->src.rowIndex = -1; pSource->src.rowIndex = -1;
...@@ -602,7 +614,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { ...@@ -602,7 +614,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
} }
} }
tsortClearOrderdSource(pHandle->pOrderedSource); tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL);
taosArrayAddAll(pHandle->pOrderedSource, pResList); taosArrayAddAll(pHandle->pOrderedSource, pResList);
taosArrayDestroy(pResList); taosArrayDestroy(pResList);
...@@ -644,7 +656,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) { ...@@ -644,7 +656,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
SSortSource* source = *pSource; SSortSource* source = *pSource;
*pSource = NULL; *pSource = NULL;
tsortClearOrderdSource(pHandle->pOrderedSource); tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL);
while (1) { while (1) {
SSDataBlock* pBlock = pHandle->fetchfp(source->param); SSDataBlock* pBlock = pHandle->fetchfp(source->param);
......
source/libs/index/src/indexFilter.c
浏览文件 @ 6811a6b2
...@@ -634,7 +634,7 @@ static FORCE_INLINE int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxF ...@@ -634,7 +634,7 @@ static FORCE_INLINE int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxF
} }
static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
int32_t code = 0; int32_t code = -1;
if (sifValidOp(node->opType) < 0) { if (sifValidOp(node->opType) < 0) {
code = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
ctx->code = code; ctx->code = code;
...@@ -654,7 +654,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { ...@@ -654,7 +654,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
SIFParam *params = NULL; SIFParam *params = NULL;
SIF_ERR_RET(sifInitOperParams(&params, node, ctx)); SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
if (params[0].status == SFLT_NOT_INDEX && (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) {
output->status = SFLT_NOT_INDEX; output->status = SFLT_NOT_INDEX;
goto _return; goto _return;
} }
...@@ -664,6 +664,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { ...@@ -664,6 +664,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
sif_func_t operFn = sifNullFunc; sif_func_t operFn = sifNullFunc;
if (!ctx->noExec) { if (!ctx->noExec) {
code = 0;
SIF_ERR_JRET(sifGetOperFn(node->opType, &operFn, &output->status)); SIF_ERR_JRET(sifGetOperFn(node->opType, &operFn, &output->status));
SIF_ERR_JRET(operFn(&params[0], nParam > 1 ? &params[1] : NULL, output)); SIF_ERR_JRET(operFn(&params[0], nParam > 1 ? &params[1] : NULL, output));
} else { } else {
...@@ -672,11 +673,17 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { ...@@ -672,11 +673,17 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
output->status = SFLT_NOT_INDEX; output->status = SFLT_NOT_INDEX;
goto _return; goto _return;
} }
code = 0;
SIF_ERR_JRET(sifGetOperFn(node->opType, &operFn, &output->status)); SIF_ERR_JRET(sifGetOperFn(node->opType, &operFn, &output->status));
} }
_return: _return:
for (int i = 0; i < nParam; i++) sifFreeParam(&params[i]); for (int i = 0; i < nParam; i++) sifFreeParam(&params[i]);
taosMemoryFree(params); taosMemoryFree(params);
if (code != 0) {
output->status = SFLT_NOT_INDEX;
} else {
output->status = SFLT_COARSE_INDEX;
}
return code; return code;
} }
...@@ -717,7 +724,7 @@ _return: ...@@ -717,7 +724,7 @@ _return:
static EDealRes sifWalkFunction(SNode *pNode, void *context) { static EDealRes sifWalkFunction(SNode *pNode, void *context) {
SFunctionNode *node = (SFunctionNode *)pNode; SFunctionNode *node = (SFunctionNode *)pNode;
SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t))}; SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t)), .status = SFLT_COARSE_INDEX};
SIFCtx *ctx = context; SIFCtx *ctx = context;
ctx->code = sifExecFunction(node, ctx, &output); ctx->code = sifExecFunction(node, ctx, &output);
...@@ -735,7 +742,7 @@ static EDealRes sifWalkFunction(SNode *pNode, void *context) { ...@@ -735,7 +742,7 @@ static EDealRes sifWalkFunction(SNode *pNode, void *context) {
static EDealRes sifWalkLogic(SNode *pNode, void *context) { static EDealRes sifWalkLogic(SNode *pNode, void *context) {
SLogicConditionNode *node = (SLogicConditionNode *)pNode; SLogicConditionNode *node = (SLogicConditionNode *)pNode;
SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t))}; SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t)), .status = SFLT_COARSE_INDEX};
SIFCtx *ctx = context; SIFCtx *ctx = context;
ctx->code = sifExecLogic(node, ctx, &output); ctx->code = sifExecLogic(node, ctx, &output);
...@@ -831,6 +838,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { ...@@ -831,6 +838,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
if (res->result != NULL) { if (res->result != NULL) {
taosArrayAddAll(pDst->result, res->result); taosArrayAddAll(pDst->result, res->result);
} }
pDst->status = res->status;
sifFreeParam(res); sifFreeParam(res);
taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES);
...@@ -887,16 +895,20 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, ...@@ -887,16 +895,20 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result,
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
SArray *output = taosArrayInit(8, sizeof(uint64_t)); SArray *output = taosArrayInit(8, sizeof(uint64_t));
SIFParam param = {.arg = *metaArg, .result = output}; SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX};
int32_t code = sifCalculate((SNode *)pFilterNode, &param); int32_t code = sifCalculate((SNode *)pFilterNode, &param);
if (code != 0) { if (code != 0) {
sifFreeParam(&param); sifFreeParam(&param);
return code; return code;
} }
if (param.status == SFLT_NOT_INDEX) {
*status = param.status;
} else {
*status = st;
}
taosArrayAddAll(result, param.result); taosArrayAddAll(result, param.result);
sifFreeParam(&param); sifFreeParam(&param);
*status = st;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
source/libs/parser/src/parInsertSml.c
浏览文件 @ 6811a6b2
...@@ -70,7 +70,7 @@ static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSche ...@@ -70,7 +70,7 @@ static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSche
SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key};
col_id_t t = lastColIdx + 1; col_id_t t = lastColIdx + 1;
col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema)); col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema));
uDebug("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols); uTrace("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols);
if (index < 0 && t > 0) { if (index < 0 && t > 0) {
index = insFindCol(&sToken, 0, t, pSchema); index = insFindCol(&sToken, 0, t, pSchema);
} }
...@@ -345,7 +345,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc ...@@ -345,7 +345,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
} }
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) {
if (errno == E2BIG) { if (errno == E2BIG) {
uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d", (int)kv->length, pColSchema->bytes); uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, pColSchema->bytes, kv->value);
buildInvalidOperationMsg(&pBuf, "value too long"); buildInvalidOperationMsg(&pBuf, "value too long");
ret = TSDB_CODE_PAR_VALUE_TOO_LONG; ret = TSDB_CODE_PAR_VALUE_TOO_LONG;
goto end; goto end;
......
source/libs/parser/src/parInsertUtil.c
浏览文件 @ 6811a6b2
...@@ -404,6 +404,7 @@ static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHa ...@@ -404,6 +404,7 @@ static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHa
int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
taosArrayPush(pVgroupList, &pVgCxt); taosArrayPush(pVgroupList, &pVgCxt);
// uDebug("td23101 2vgId:%d, uid:%" PRIu64, pVgCxt->vgId, pTableCxt->pMeta->uid);
*pOutput = pVgCxt; *pOutput = pVgCxt;
} else { } else {
insDestroyVgroupDataCxt(pVgCxt); insDestroyVgroupDataCxt(pVgCxt);
...@@ -546,6 +547,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, ...@@ -546,6 +547,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList,
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData);
code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
// uError("td23101 3vgId:%d, numEps:%d", src->vgId, dst->vg.epSet.numOfEps);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size);
......
source/util/src/tlog.c
浏览文件 @ 6811a6b2
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
#include "tjson.h" #include "tjson.h"
#include "tglobal.h" #include "tglobal.h"
#define LOG_MAX_LINE_SIZE (1024) #define LOG_MAX_LINE_SIZE (10024)
#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3)
#define LOG_MAX_LINE_DUMP_SIZE (1024 * 1024) #define LOG_MAX_LINE_DUMP_SIZE (1024 * 1024)
#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 3) #define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 3)
......
tests/parallel_test/cases.task
浏览文件 @ 6811a6b2
...@@ -245,6 +245,7 @@ ...@@ -245,6 +245,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py
...@@ -469,6 +470,7 @@ ...@@ -469,6 +470,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2
...@@ -563,6 +565,7 @@ ...@@ -563,6 +565,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3
...@@ -658,6 +661,7 @@ ...@@ -658,6 +661,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4
...@@ -849,6 +853,7 @@ ...@@ -849,6 +853,7 @@
,,y,script,./test.sh -f tsim/parser/topbot.sim ,,y,script,./test.sh -f tsim/parser/topbot.sim
,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim ,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim
,,y,script,./test.sh -f tsim/parser/slimit_limit.sim ,,y,script,./test.sh -f tsim/parser/slimit_limit.sim
,,y,script,./test.sh -f tsim/parser/table_merge_limit.sim
,,y,script,./test.sh -f tsim/query/tagLikeFilter.sim ,,y,script,./test.sh -f tsim/query/tagLikeFilter.sim
,,y,script,./test.sh -f tsim/query/charScalarFunction.sim ,,y,script,./test.sh -f tsim/query/charScalarFunction.sim
,,y,script,./test.sh -f tsim/query/explain.sim ,,y,script,./test.sh -f tsim/query/explain.sim
......
tests/parallel_test/container_build.sh
浏览文件 @ 6811a6b2
...@@ -68,8 +68,8 @@ docker run \ ...@@ -68,8 +68,8 @@ docker run \
-v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
-v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=true;make -j || exit 1"
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true;make -j || exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
if [[ -d ${WORKDIR}/debugNoSan ]] ;then if [[ -d ${WORKDIR}/debugNoSan ]] ;then
echo "delete ${WORKDIR}/debugNoSan" echo "delete ${WORKDIR}/debugNoSan"
...@@ -97,7 +97,7 @@ docker run \ ...@@ -97,7 +97,7 @@ docker run \
-v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true;make -j || exit 1 " --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=true;make -j || exit 1 "
mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan
......
tests/pytest/util/common.py
浏览文件 @ 6811a6b2
...@@ -739,6 +739,11 @@ class TDCom: ...@@ -739,6 +739,11 @@ class TDCom:
else: else:
os.system("unset LD_PRELOAD; pkill %s " % processorName) os.system("unset LD_PRELOAD; pkill %s " % processorName)
def gen_tag_col_str(self, gen_type, data_type, count):
"""
gen multi tags or cols by gen_type
"""
return ','.join(map(lambda i: f'{gen_type}{i} {data_type}', range(count)))
def is_json(msg): def is_json(msg):
if isinstance(msg, str): if isinstance(msg, str):
...@@ -775,4 +780,5 @@ def dict2toml(in_dict: dict, file:str): ...@@ -775,4 +780,5 @@ def dict2toml(in_dict: dict, file:str):
with open(file, 'w') as f: with open(file, 'w') as f:
toml.dump(in_dict, f) toml.dump(in_dict, f)
tdCom = TDCom() tdCom = TDCom()
tests/script/coverage_test.sh
浏览文件 @ 6811a6b2
...@@ -12,6 +12,7 @@ fi ...@@ -12,6 +12,7 @@ fi
today=`date +"%Y%m%d"` today=`date +"%Y%m%d"`
TDENGINE_DIR=/root/TDengine TDENGINE_DIR=/root/TDengine
JDBC_DIR=/root/taos-connector-jdbc JDBC_DIR=/root/taos-connector-jdbc
TAOSKEEPER_DIR=/root/taoskeeper
TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
# Color setting # Color setting
...@@ -171,6 +172,24 @@ function runJDBCCases() { ...@@ -171,6 +172,24 @@ function runJDBCCases() {
echo -e "### JDBC test result: $summary ###" | tee -a $TDENGINE_COVERAGE_REPORT echo -e "### JDBC test result: $summary ###" | tee -a $TDENGINE_COVERAGE_REPORT
} }
function runTaosKeeperCases() {
echo "=== Run taoskeeper cases ==="
cd $TAOSKEEPER_DIR
git checkout -- .
git reset --hard HEAD
git checkout master
git pull
stopTaosd
stopTaosadapter
taosd -c /etc/taos >> /dev/null 2>&1 &
taosadapter >> /dev/null 2>&1 &
go mod tidy && go test -v ./...
}
function runTest() { function runTest() {
echo "run Test" echo "run Test"
...@@ -182,6 +201,7 @@ function runTest() { ...@@ -182,6 +201,7 @@ function runTest() {
runSimCases runSimCases
runPythonCases runPythonCases
runJDBCCases runJDBCCases
runTaosKeeperCases
stopTaosd stopTaosd
cd $TDENGINE_DIR/tests/script cd $TDENGINE_DIR/tests/script
...@@ -199,7 +219,7 @@ function lcovFunc { ...@@ -199,7 +219,7 @@ function lcovFunc {
lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
# remove exclude paths # remove exclude paths
if [ "$branch" == "3.0" ]; then if [ "$branch" == "main" ] ; then
lcov --remove coverage.info \ lcov --remove coverage.info \
'*/contrib/*' '*/tests/*' '*/test/*' '*/tools/*' '*/libs/sync/*'\ '*/contrib/*' '*/tests/*' '*/test/*' '*/tools/*' '*/libs/sync/*'\
'*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\ '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\
...@@ -209,6 +229,8 @@ function lcovFunc { ...@@ -209,6 +229,8 @@ function lcovFunc {
'*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' '*/catalog.c'\ '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' '*/catalog.c'\
'*/tqSnapshot.c' '*/tsdbSnapshot.c''*/metaSnapshot.c' '*/smaSnapshot.c' '*/tqOffsetSnapshot.c'\ '*/tqSnapshot.c' '*/tsdbSnapshot.c''*/metaSnapshot.c' '*/smaSnapshot.c' '*/tqOffsetSnapshot.c'\
'*/vnodeSnapshot.c' '*/metaSnapshot.c' '*/tsdbSnapshot.c' '*/mndGrant.c' '*/mndSnode.c' '*/streamRecover.c'\ '*/vnodeSnapshot.c' '*/metaSnapshot.c' '*/tsdbSnapshot.c' '*/mndGrant.c' '*/mndSnode.c' '*/streamRecover.c'\
'*/osAtomic.c' '*/osDir.c' '*/osFile.c' '*/osMath.c' '*/osSignal.c' '*/osSleep.c' '*/osString.c' '*/osSystem.c'\
'*/osThread.c' '*/osTime.c' '*/osTimezone.c' \
--rc lcov_branch_coverage=1 -o coverage.info --rc lcov_branch_coverage=1 -o coverage.info
else else
lcov --remove coverage.info \ lcov --remove coverage.info \
......
tests/script/tsim/parser/table_merge_limit.sim 0 → 100644
浏览文件 @ 6811a6b2
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
$dbPrefix = m_fl_db
$tbPrefix = m_fl_tb
$mtPrefix = m_fl_mt
$tbNum = 2
$rowNum = 513
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000
print ========== fill.sim
$i = 0
$db = $dbPrefix . $i
$mt = $mtPrefix . $i
sql drop database $db -x step1
step1:
sql create database $db vgroups 1
sql use $db
sql create table $mt (ts timestamp, c1 int) tags(tgcol int)
$i = 0
$ts = $ts0
while $i < $tbNum
$tb = $tbPrefix . $i
sql create table $tb using $mt tags( $i )
$x = 0
while $x < $rowNum
$xs = $x * $delta
$ts = $ts0 + $xs
sql insert into $tb values ( $ts , $x )
$x = $x + 1
endw
$i = $i + 1
endw
sql select * from $mt order by ts limit 10
if $rows != 10 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
tests/system-test/0-others/check_assert.py 0 → 100644
浏览文件 @ 6811a6b2
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
'''
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
'''
import sys
import random
import os
#define code
NO_FOUND = 0 # not found assert or ASSERT
FOUND_OK = 1 # found ASSERT and valid usage
FOUND_NOIF = 2 # found ASSERT but no if like ASSERT(...)
FOUND_LOWER = 3 # found assert write with lower letters
FOUND_HAVENOT = 4 # found ASSERT have if but have not like if(!ASSERT)
code_strs = ["not found", "valid", "found but no if", "lower assert","found but have not"]
#
# check assert
#
class CheckAssert:
def __init__(self):
self.files = 0
self.total = [0,0,0,0,0]
def wholeMatched(self, line, pos, n):
before = False
after = False
if pos != -1:
if pos == 0 or line[pos-1] == ' ' or line[pos-1] == '!' or line[pos-1] == '(':
before = True
if n > pos + 6 + 1:
last = line[pos+6]
if last == ' ' or last == '(':
after = True
if before and after:
return True
else:
return False
def parseLine(self, line):
# see FOUND_xxx define below
code = 0
n = len(line)
if(n < 7):
return NO_FOUND
s1 = line.find("//") # commit line
# assert
pos = line.find("assert")
if pos != -1:
if self.wholeMatched(line, pos, n):
# comment line check
if not (s1 != -1 and s1 < pos):
return FOUND_LOWER
# ASSERT
pos = 0
while pos < n:
pos = line.find("ASSERT", pos)
#print(f" pos={pos} {line}")
if pos == -1:
return NO_FOUND
if self.wholeMatched(line, pos, n):
break
# next
pos += 6
#
# found
#
# comment line
if s1 != -1 and s1 < pos:
return NO_FOUND
# check if
s2 = line.find("if")
if s2 == -1 or s2 > pos or pos - s2 > 5:
return FOUND_NOIF
s3 = line.find("!")
# if(!ASSERT
if s3 > s2 and s3 < pos:
return FOUND_HAVENOT
return FOUND_OK
def checkFile(self, pathfile):
# check .h .c
ext = pathfile[-2:].lower()
if ext != ".h" and ext != ".c":
return
print(" check file %s"%pathfile)
self.files += 1
err = 0
ok = 0
i = 0
# read file context
with open(pathfile, "r") as fp:
lines = fp.readlines()
for line in lines:
i += 1
code = self.parseLine(line)
self.total[code] += 1
if code == FOUND_OK:
ok += 1
if code != NO_FOUND and code != FOUND_OK:
err += 1
if code != NO_FOUND:
print(f" line: {i} code: {code} {line}")
# parse end output total
if err > 0 or ok > 0:
print(f" found problem: {err} \n")
def scanPath(self, path):
#print(" check path %s"%path)
ignores = ["/test/"]
for ignore in ignores:
if ignore in path:
print(f" ignore {path} keyword: {ignore}")
return
with os.scandir(path) as childs:
for child in childs:
if child.is_file():
self.checkFile(os.path.join(path, child.name))
elif child.is_dir():
self.scanPath(child.path)
def doCheck(self, path):
print(f" start check path:{path}")
self.scanPath(path)
# print total
print("\n")
print(f" --------------- total ({self.files} files)--------------")
for i in range(5):
print(f" code : {i} num: {self.total[i]} ({code_strs[i]})")
print(" --------------- end ----------------")
print(f"\n")
#
# main function
#
if __name__ == "__main__":
print(" hello, welcome to use check assert tools 1.0.")
path = os.path.dirname(os.path.realpath(__file__))
label = "TDengine"
pos = path.find(label)
if pos != -1:
pos += len(label) + 1
src = path[0:pos] + "source"
else:
src = path
checker = CheckAssert()
checker.doCheck(src)
print(" check assert finished")
'''
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.checker = CheckAssert()
# run
def run(self):
# calc
selfPath = os.path.dirname(os.path.realpath(__file__))
projPath = ""
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
src = self.projPath + "src/"
self.checker.checkAssert(src)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
'''
\ No newline at end of file
tests/system-test/0-others/compatibility.py
浏览文件 @ 6811a6b2
...@@ -97,7 +97,7 @@ class TDTestCase: ...@@ -97,7 +97,7 @@ class TDTestCase:
def buildTaosd(self,bPath): def buildTaosd(self,bPath):
# os.system(f"mv {bPath}/build_bak {bPath}/build ") # os.system(f"mv {bPath}/build_bak {bPath}/build ")
os.system(f" cd {bPath} && make install ") os.system(f" cd {bPath} ")
def run(self): def run(self):
...@@ -142,6 +142,10 @@ class TDTestCase: ...@@ -142,6 +142,10 @@ class TDTestCase:
tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ")
os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y")
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '")
cmd = f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taos -h localhost ;"
if os.system(cmd) == 0:
raise Exception("failed to execute system command. cmd: %s" % cmd)
os.system("pkill taosd") # make sure all the data are saved in disk. os.system("pkill taosd") # make sure all the data are saved in disk.
self.checkProcessPid("taosd") self.checkProcessPid("taosd")
...@@ -152,8 +156,10 @@ class TDTestCase: ...@@ -152,8 +156,10 @@ class TDTestCase:
sleep(1) sleep(1)
tdsql=tdCom.newTdSql() tdsql=tdCom.newTdSql()
print(tdsql) print(tdsql)
cmd = f" LD_LIBRARY_PATH=/usr/lib taos -h localhost ;"
if os.system(cmd) == 0:
raise Exception("failed to execute system command. cmd: %s" % cmd)
tdsql.query(f"SELECT SERVER_VERSION();") tdsql.query(f"SELECT SERVER_VERSION();")
nowServerVersion=tdsql.queryResult[0][0] nowServerVersion=tdsql.queryResult[0][0]
tdLog.info(f"New server version is {nowServerVersion}") tdLog.info(f"New server version is {nowServerVersion}")
......
tests/system-test/0-others/information_schema.py
浏览文件 @ 6811a6b2
...@@ -93,7 +93,6 @@ class TDTestCase: ...@@ -93,7 +93,6 @@ class TDTestCase:
tdSql.checkEqual(i[2],len(self.perf_list)) tdSql.checkEqual(i[2],len(self.perf_list))
tdSql.execute('create table db1.ntb (ts timestamp,c0 int)') tdSql.execute('create table db1.ntb (ts timestamp,c0 int)')
tdSql.query(f'select db_name, count(*) from information_schema.ins_tables group by db_name') tdSql.query(f'select db_name, count(*) from information_schema.ins_tables group by db_name')
print(tdSql.queryResult)
for i in tdSql.queryResult: for i in tdSql.queryResult:
if i[0].lower() == 'information_schema': if i[0].lower() == 'information_schema':
tdSql.checkEqual(i[1],len(self.ins_list)) tdSql.checkEqual(i[1],len(self.ins_list))
...@@ -101,18 +100,78 @@ class TDTestCase: ...@@ -101,18 +100,78 @@ class TDTestCase:
tdSql.checkEqual(i[1],len(self.perf_list)) tdSql.checkEqual(i[1],len(self.perf_list))
elif i[0].lower() == self.dbname: elif i[0].lower() == self.dbname:
tdSql.checkEqual(i[1],self.tbnum+1) tdSql.checkEqual(i[1],self.tbnum+1)
def ins_col_check_4096(self):
tdSql.execute('create database db3 vgroups 2 replica 1')
col_str = tdCom.gen_tag_col_str("col", "int",4094)
tdSql.execute(f'create stable if not exists db3.stb (col_ts timestamp, {col_str}) tags (t1 int)')
for i in range(100):
tdLog.info(f"create table db3.ctb{i} using db3.stb tags({i})")
tdSql.execute(f"create table db3.ctb{i} using db3.stb tags({i})")
col_value_str = '1, ' * 4093 + '1'
tdSql.execute(f"insert into db3.ctb{i} values(now,{col_value_str})(now+1s,{col_value_str})(now+2s,{col_value_str})(now+3s,{col_value_str})")
tdSql.query("select * from information_schema.ins_columns")
tdSql.execute('drop database db3')
def ins_stable_check(self):
tdSql.execute('create database db3 vgroups 2 replica 1')
tbnum = 10
ctbnum = 10
for i in range(tbnum):
tdSql.execute(f'create stable db3.stb_{i} (ts timestamp,c0 int) tags(t0 int)')
tdSql.execute(f'create table db3.ntb_{i} (ts timestamp,c0 int)')
for j in range(ctbnum):
tdSql.execute(f"create table db3.ctb_{i}_{j} using db3.stb_{i} tags({j})")
tdSql.query("select stable_name,count(table_name) from information_schema.ins_tables where db_name = 'db3' group by stable_name order by stable_name")
result = tdSql.queryResult
for i in range(len(result)):
if result[i][0] == None:
tdSql.checkEqual(result[0][1],tbnum)
else:
tdSql.checkEqual(result[i][0],f'stb_{i-1}')
tdSql.checkEqual(result[i][1],ctbnum)
def ins_columns_check(self): def ins_columns_check(self):
tdSql.execute('create database db2 vgroups 2 replica 1') tdSql.execute('drop database if exists db2')
tdSql.execute('create table db2.stb2 (ts timestamp,c0 int,c1 int, c2 double, c3 float, c4 binary(1000), c5 nchar(100),c7 bigint, c8 bool, c9 smallint) tags(t0 int)') tdSql.execute('create database if not exists db2 vgroups 1 replica 1')
for i in range(2000): for i in range (5):
tdSql.execute("create table db2.ctb%d using db2.stb2 tags(%d)" %(i,i)) self.stb4096 = 'create table db2.stb%d (ts timestamp' % (i)
tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type="CHILD_TABLE"') for j in range (4094 - i):
tdSql.checkEqual(20000,len(tdSql.queryResult)) self.stb4096 += ', c%d int' % (j)
print("number of ins_columns of child table in db2 is %s" % len(tdSql.queryResult)) self.stb4096 += ') tags (t1 int)'
tdSql.execute(self.stb4096)
for k in range(10):
tdSql.execute("create table db2.ctb_%d_%dc using db2.stb%d tags(%d)" %(i,k,i,k))
for t in range (2):
tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type=="SUPER_TABLE"')
tdSql.checkEqual(20465,len(tdSql.queryResult))
for t in range (2):
tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type=="CHILD_TABLE"')
tdSql.checkEqual(204650,len(tdSql.queryResult))
for i in range (5):
self.ntb4096 = 'create table db2.ntb%d (ts timestamp' % (i)
for j in range (4095 - i):
self.ntb4096 += ', c%d binary(10)' % (j)
self.ntb4096 += ')'
tdSql.execute(self.ntb4096)
for t in range (2):
tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type=="NORMAL_TABLE"')
tdSql.checkEqual(20470,len(tdSql.queryResult))
def run(self): def run(self):
self.prepare_data() self.prepare_data()
self.count_check() self.count_check()
self.ins_columns_check() self.ins_columns_check()
# self.ins_col_check_4096()
self.ins_stable_check()
def stop(self): def stop(self):
tdSql.close() tdSql.close()
tdLog.success("%s successfully executed" % __file__) tdLog.success("%s successfully executed" % __file__)
......
tests/system-test/0-others/tag_index_basic.py
浏览文件 @ 6811a6b2
...@@ -24,7 +24,7 @@ class TDTestCase: ...@@ -24,7 +24,7 @@ class TDTestCase:
def init(self, conn, logSql, replicaVar=1): def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar) self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__) tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), True)
self.setsql = TDSetSql() self.setsql = TDSetSql()
self.column_dict = { self.column_dict = {
'ts': 'timestamp', 'ts': 'timestamp',
......
tests/system-test/2-query/last_row_interval.py 0 → 100644
浏览文件 @ 6811a6b2
因为 它太大了无法显示 source diff 。你可以改为 查看blob。
tests/system-test/2-query/tsbsQuery.py
浏览文件 @ 6811a6b2
...@@ -211,15 +211,16 @@ class TDTestCase: ...@@ -211,15 +211,16 @@ class TDTestCase:
tdSql.query(f"select model,state_changed,count(state_changed) FROM (SELECT model,diff(broken_down) AS state_changed FROM (SELECT _wstart,model,cast(cast(floor(2*(sum(nzs)/count(nzs))) as bool) as int) AS broken_down FROM (SELECT ts,model, cast(cast(status as bool) as int) AS nzs from {dbname}.diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' ) WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition BY model interval(10m)) partition BY model) where state_changed =1 partition BY model,state_changed ;") tdSql.query(f"select model,state_changed,count(state_changed) FROM (SELECT model,diff(broken_down) AS state_changed FROM (SELECT _wstart,model,cast(cast(floor(2*(sum(nzs)/count(nzs))) as bool) as int) AS broken_down FROM (SELECT ts,model, cast(cast(status as bool) as int) AS nzs from {dbname}.diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' ) WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition BY model interval(10m)) partition BY model) where state_changed =1 partition BY model,state_changed ;")
sql=f"select model,ctc from (SELECT model,count(state_changed) as ctc FROM (SELECT model,diff(broken_down) AS state_changed FROM (SELECT model,cast(cast(floor(2*(sum(nzs)/count(nzs))) as bool) as int) AS broken_down FROM (SELECT ts,model, cast(cast(status as bool) as int) AS nzs from {dbname}.diagnostics WHERE ts >= 1451606400000 AND ts < 1451952001000 ) WHERE ts >= 1451606400000 AND ts < 1451952001000 partition BY model interval(10m)) partition BY model) WHERE state_changed = 1 partition BY model )where model is null;" sql=f"SELECT model,count(state_changed) FROM (SELECT _rowts,model,diff(broken_down) AS state_changed FROM (SELECT ts,model,tb,cast(cast(floor(2*(nzs)) as bool) as int) AS broken_down FROM (SELECT _wstart as ts,model,tbname as tb, sum(cast(cast(status as bool) as int))/count(cast(cast(status as bool) as int)) AS nzs FROM {dbname}.diagnostics WHERE ts >= 1451606400000 AND ts < 1451952001000 partition BY tbname,model interval(10m))order by ts) partition BY tb,model ) WHERE state_changed = 1 partition BY model;"
tdSql.query(f"{sql}")
# for i in range(2): tdSql.checkRows(46)
# tdSql.query("%s"%sql) for i in range(2):
# quertR1=tdSql.queryResult tdSql.query("%s"%sql)
# for j in range(50): quertR1=tdSql.queryResult
# tdSql.query("%s"%sql) for j in range(50):
# quertR2=tdSql.queryResult tdSql.query("%s"%sql)
# assert quertR1 == quertR2 , "%s != %s ,The results of multiple queries are different" %(quertR1,quertR2) quertR2=tdSql.queryResult
assert quertR1 == quertR2 , "%s != %s ,The results of multiple queries are different" %(quertR1,quertR2)
#it's already supported: #it's already supported:
......
tools/shell/inc/shellInt.h
浏览文件 @ 6811a6b2
...@@ -28,6 +28,10 @@ ...@@ -28,6 +28,10 @@
#ifdef WEBSOCKET #ifdef WEBSOCKET
#include "taosws.h" #include "taosws.h"
#define SHELL_WS_TIMEOUT 30
#define SHELL_WS_DSN_BUFF 256
#define SHELL_WS_DSN_MASK 10
#endif #endif
#define SHELL_MAX_HISTORY_SIZE 1000 #define SHELL_MAX_HISTORY_SIZE 1000
...@@ -99,7 +103,7 @@ typedef struct { ...@@ -99,7 +103,7 @@ typedef struct {
bool exit; bool exit;
#ifdef WEBSOCKET #ifdef WEBSOCKET
WS_TAOS* ws_conn; WS_TAOS* ws_conn;
bool stop_query; bool stop_query;
#endif #endif
} SShellObj; } SShellObj;
...@@ -139,7 +143,7 @@ void shellExit(); ...@@ -139,7 +143,7 @@ void shellExit();
void shellTestNetWork(); void shellTestNetWork();
#ifdef WEBSOCKET #ifdef WEBSOCKET
void shellCheckConnectMode(); void shellCheckConnectMode();
// shellWebsocket.c // shellWebsocket.c
int shell_conn_ws_server(bool first); int shell_conn_ws_server(bool first);
int32_t shell_run_websocket(); int32_t shell_run_websocket();
......
tools/shell/src/shellArguments.c
浏览文件 @ 6811a6b2
...@@ -60,7 +60,7 @@ ...@@ -60,7 +60,7 @@
#ifdef WEBSOCKET #ifdef WEBSOCKET
#define SHELL_DSN "The dsn to use when connecting to cloud server." #define SHELL_DSN "The dsn to use when connecting to cloud server."
#define SHELL_REST "Use restful mode when connecting." #define SHELL_REST "Use restful mode when connecting."
#define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 10." #define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30."
#endif #endif
static int32_t shellParseSingleOpt(int32_t key, char *arg); static int32_t shellParseSingleOpt(int32_t key, char *arg);
...@@ -127,7 +127,7 @@ static struct argp_option shellOptions[] = { ...@@ -127,7 +127,7 @@ static struct argp_option shellOptions[] = {
#ifdef WEBSOCKET #ifdef WEBSOCKET
{"dsn", 'E', "DSN", 0, SHELL_DSN}, {"dsn", 'E', "DSN", 0, SHELL_DSN},
{"restful", 'R', 0, 0, SHELL_REST}, {"restful", 'R', 0, 0, SHELL_REST},
{"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT},
#endif #endif
{"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM},
{0}, {0},
...@@ -223,9 +223,9 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { ...@@ -223,9 +223,9 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) {
pArgs->dsn = arg; pArgs->dsn = arg;
pArgs->cloud = true; pArgs->cloud = true;
break; break;
case 'T': case 'T':
pArgs->timeout = atoi(arg); pArgs->timeout = atoi(arg);
break; break;
#endif #endif
case 'V': case 'V':
pArgs->is_version = true; pArgs->is_version = true;
...@@ -246,7 +246,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { ...@@ -246,7 +246,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
SShellArgs *pArgs = &shell.args; SShellArgs *pArgs = &shell.args;
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0
|| strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) {
shellParseSingleOpt('?', NULL); shellParseSingleOpt('?', NULL);
return 0; return 0;
} }
...@@ -262,12 +263,14 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { ...@@ -262,12 +263,14 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
return -1; return -1;
} }
if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' || if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u'
key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' || key[1] == 'a' || key[1] == 'c' || key[1] == 's'
|| key[1] == 'f' || key[1] == 'd' || key[1] == 'w'
|| key[1] == 'n' || key[1] == 'l' || key[1] == 'N'
#ifdef WEBSOCKET #ifdef WEBSOCKET
|| key[1] == 'E' || key[1] == 'T' || key[1] == 'E' || key[1] == 'T'
#endif #endif
) { ) {
if (i + 1 >= argc) { if (i + 1 >= argc) {
fprintf(stderr, "option %s requires an argument\r\n", key); fprintf(stderr, "option %s requires an argument\r\n", key);
return -1; return -1;
...@@ -279,12 +282,14 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { ...@@ -279,12 +282,14 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
} }
shellParseSingleOpt(key[1], val); shellParseSingleOpt(key[1], val);
i++; i++;
} else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C'
key[1] == 't' || key[1] == 'V' || key[1] == '?' || key[1] == 1 || key[1] == 'r' || key[1] == 'k'
|| key[1] == 't' || key[1] == 'V'
|| key[1] == '?' || key[1] == 1
#ifdef WEBSOCKET #ifdef WEBSOCKET
||key[1] == 'R' ||key[1] == 'R'
#endif #endif
) { ) {
shellParseSingleOpt(key[1], NULL); shellParseSingleOpt(key[1], NULL);
} else { } else {
fprintf(stderr, "invalid option %s\r\n", key); fprintf(stderr, "invalid option %s\r\n", key);
...@@ -406,7 +411,7 @@ static int32_t shellCheckArgs() { ...@@ -406,7 +411,7 @@ static int32_t shellCheckArgs() {
int32_t shellParseArgs(int32_t argc, char *argv[]) { int32_t shellParseArgs(int32_t argc, char *argv[]) {
shellInitArgs(argc, argv); shellInitArgs(argc, argv);
shell.info.clientVersion = shell.info.clientVersion =
"Welcome to the %s Command Line Interface, Client Version:%s\r\n" "Welcome to the %s Command Line Interface, Client Version:%s\r\n"
"Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n";
strcpy(shell.info.cusName, cusName); strcpy(shell.info.cusName, cusName);
......
tools/shell/src/shellEngine.c
浏览文件 @ 6811a6b2
...@@ -102,10 +102,10 @@ int32_t shellRunSingleCommand(char *command) { ...@@ -102,10 +102,10 @@ int32_t shellRunSingleCommand(char *command) {
} }
#ifdef WEBSOCKET #ifdef WEBSOCKET
if (shell.args.restful || shell.args.cloud) { if (shell.args.restful || shell.args.cloud) {
shellRunSingleCommandWebsocketImp(command); shellRunSingleCommandWebsocketImp(command);
} else { } else {
#endif #endif
shellRunSingleCommandImp(command); shellRunSingleCommandImp(command);
#ifdef WEBSOCKET #ifdef WEBSOCKET
} }
#endif #endif
...@@ -1025,15 +1025,15 @@ void *shellCancelHandler(void *arg) { ...@@ -1025,15 +1025,15 @@ void *shellCancelHandler(void *arg) {
} }
#ifdef WEBSOCKET #ifdef WEBSOCKET
if (shell.args.restful || shell.args.cloud) { if (shell.args.restful || shell.args.cloud) {
shell.stop_query = true; shell.stop_query = true;
} else { } else {
#endif #endif
if (shell.conn) { if (shell.conn) {
taos_kill_query(shell.conn); taos_kill_query(shell.conn);
} }
#ifdef WEBSOCKET #ifdef WEBSOCKET
} }
#endif #endif
#ifdef WINDOWS #ifdef WINDOWS
printf("\n%s", shell.info.promptHeader); printf("\n%s", shell.info.promptHeader);
...@@ -1083,21 +1083,21 @@ int32_t shellExecute() { ...@@ -1083,21 +1083,21 @@ int32_t shellExecute() {
SShellArgs *pArgs = &shell.args; SShellArgs *pArgs = &shell.args;
#ifdef WEBSOCKET #ifdef WEBSOCKET
if (shell.args.restful || shell.args.cloud) { if (shell.args.restful || shell.args.cloud) {
if (shell_conn_ws_server(1)) { if (shell_conn_ws_server(1)) {
return -1; return -1;
} }
} else { } else {
#endif #endif
if (shell.args.auth == NULL) { if (shell.args.auth == NULL) {
shell.conn = taos_connect(pArgs->host, pArgs->user, pArgs->password, pArgs->database, pArgs->port); shell.conn = taos_connect(pArgs->host, pArgs->user, pArgs->password, pArgs->database, pArgs->port);
} else { } else {
shell.conn = taos_connect_auth(pArgs->host, pArgs->user, pArgs->auth, pArgs->database, pArgs->port); shell.conn = taos_connect_auth(pArgs->host, pArgs->user, pArgs->auth, pArgs->database, pArgs->port);
} }
if (shell.conn == NULL) { if (shell.conn == NULL) {
fflush(stdout); fflush(stdout);
return -1; return -1;
} }
#ifdef WEBSOCKET #ifdef WEBSOCKET
} }
#endif #endif
...@@ -1118,13 +1118,13 @@ int32_t shellExecute() { ...@@ -1118,13 +1118,13 @@ int32_t shellExecute() {
shellSourceFile(pArgs->file); shellSourceFile(pArgs->file);
} }
#ifdef WEBSOCKET #ifdef WEBSOCKET
if (shell.args.restful || shell.args.cloud) { if (shell.args.restful || shell.args.cloud) {
ws_close(shell.ws_conn); ws_close(shell.ws_conn);
} else { } else {
#endif #endif
taos_close(shell.conn); taos_close(shell.conn);
#ifdef WEBSOCKET #ifdef WEBSOCKET
} }
#endif #endif
shellWriteHistory(); shellWriteHistory();
...@@ -1148,9 +1148,9 @@ int32_t shellExecute() { ...@@ -1148,9 +1148,9 @@ int32_t shellExecute() {
if (!shell.args.restful && !shell.args.cloud) { if (!shell.args.restful && !shell.args.cloud) {
#endif #endif
#ifndef WINDOWS #ifndef WINDOWS
printfIntroduction(); printfIntroduction();
#endif #endif
shellGetGrantInfo(); shellGetGrantInfo();
#ifdef WEBSOCKET #ifdef WEBSOCKET
} }
#endif #endif
......
tools/shell/src/shellMain.c
浏览文件 @ 6811a6b2
...@@ -45,7 +45,7 @@ void shellCrashHandler(int signum, void *sigInfo, void *context) { ...@@ -45,7 +45,7 @@ void shellCrashHandler(int signum, void *sigInfo, void *context) {
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
shell.exit = false; shell.exit = false;
#ifdef WEBSOCKET #ifdef WEBSOCKET
shell.args.timeout = 10; shell.args.timeout = SHELL_WS_TIMEOUT;
shell.args.cloud = true; shell.args.cloud = true;
#endif #endif
......
tools/shell/src/shellWebsocket.c
浏览文件 @ 6811a6b2
...@@ -14,22 +14,51 @@ ...@@ -14,22 +14,51 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifdef WEBSOCKET #ifdef WEBSOCKET
#include "taosws.h" #include <taosws.h>
#include "shellInt.h" #include <shellInt.h>
int shell_conn_ws_server(bool first) { int shell_conn_ws_server(bool first) {
shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); char cuttedDsn[SHELL_WS_DSN_BUFF] = {0};
if (!shell.ws_conn) { int dsnLen = strlen(shell.args.dsn);
fprintf(stderr, "failed to connect %s, reason: %s\n", snprintf(cuttedDsn,
shell.args.dsn, ws_errstr(NULL)); ((dsnLen-SHELL_WS_DSN_MASK) > SHELL_WS_DSN_BUFF)?
SHELL_WS_DSN_BUFF:(dsnLen-SHELL_WS_DSN_MASK),
"%s", shell.args.dsn);
fprintf(stdout, "trying to connect %s*** ", cuttedDsn);
fflush(stdout);
for (int i = 0; i < shell.args.timeout; i++) {
shell.ws_conn = ws_connect_with_dsn(shell.args.dsn);
if (NULL == shell.ws_conn) {
int errNo = ws_errno(NULL);
if (0xE001 == errNo) {
fprintf(stdout, ".");
fflush(stdout);
taosMsleep(1000); // sleep 1 second then try again
continue;
} else {
fprintf(stderr, "\nfailed to connect %s***, reason: %s\n",
cuttedDsn, ws_errstr(NULL));
return -1;
}
} else {
break;
}
}
if (NULL == shell.ws_conn) {
fprintf(stdout, "\n timeout\n");
fprintf(stderr, "\nfailed to connect %s***, reason: %s\n",
cuttedDsn, ws_errstr(NULL));
return -1; return -1;
} else {
fprintf(stdout, "\n");
} }
if (first && shell.args.restful) { if (first && shell.args.restful) {
fprintf(stdout, "successfully connect to %s\n\n", fprintf(stdout, "successfully connected to %s\n\n",
shell.args.dsn); shell.args.dsn);
} else if (first && shell.args.cloud) { } else if (first && shell.args.cloud) {
fprintf(stdout, "successfully connect to cloud service\n"); fprintf(stdout, "successfully connected to cloud service\n");
} }
fflush(stdout);
return 0; return 0;
} }
...@@ -118,7 +147,8 @@ static int verticalPrintWebsocket(WS_RES* wres, double* pexecute_time) { ...@@ -118,7 +147,8 @@ static int verticalPrintWebsocket(WS_RES* wres, double* pexecute_time) {
return numOfRows; return numOfRows;
} }
static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute_time) { static int dumpWebsocketToFile(const char* fname, WS_RES* wres,
double* pexecute_time) {
char fullname[PATH_MAX] = {0}; char fullname[PATH_MAX] = {0};
if (taosExpandDir(fname, fullname, PATH_MAX) != 0) { if (taosExpandDir(fname, fullname, PATH_MAX) != 0) {
tstrncpy(fullname, fname, PATH_MAX); tstrncpy(fullname, fname, PATH_MAX);
...@@ -150,7 +180,7 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute ...@@ -150,7 +180,7 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute
} }
taosFprintfFile(pFile, "%s", fields[col].name); taosFprintfFile(pFile, "%s", fields[col].name);
} }
taosFprintfFile(pFile, "\r\n"); taosFprintfFile(pFile, "\r\n");
do { do {
uint8_t ty; uint8_t ty;
uint32_t len; uint32_t len;
...@@ -161,7 +191,8 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute ...@@ -161,7 +191,8 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute
taosFprintfFile(pFile, ","); taosFprintfFile(pFile, ",");
} }
const void *value = ws_get_value_in_block(wres, i, j, &ty, &len); const void *value = ws_get_value_in_block(wres, i, j, &ty, &len);
shellDumpFieldToFile(pFile, (const char*)value, fields + j, len, precision); shellDumpFieldToFile(pFile, (const char*)value,
fields + j, len, precision);
} }
taosFprintfFile(pFile, "\r\n"); taosFprintfFile(pFile, "\r\n");
} }
...@@ -171,7 +202,9 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute ...@@ -171,7 +202,9 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute
return numOfRows; return numOfRows;
} }
static int shellDumpWebsocket(WS_RES *wres, char *fname, int *error_no, bool vertical, double* pexecute_time) { static int shellDumpWebsocket(WS_RES *wres, char *fname,
int *error_no, bool vertical,
double* pexecute_time) {
int numOfRows = 0; int numOfRows = 0;
if (fname != NULL) { if (fname != NULL) {
numOfRows = dumpWebsocketToFile(fname, wres, pexecute_time); numOfRows = dumpWebsocketToFile(fname, wres, pexecute_time);
...@@ -227,13 +260,16 @@ void shellRunSingleCommandWebsocketImp(char *command) { ...@@ -227,13 +260,16 @@ void shellRunSingleCommandWebsocketImp(char *command) {
// if it's not a ws connection error // if it's not a ws connection error
if (TSDB_CODE_WS_DSN_ERROR != (code&TSDB_CODE_WS_DSN_ERROR)) { if (TSDB_CODE_WS_DSN_ERROR != (code&TSDB_CODE_WS_DSN_ERROR)) {
et = taosGetTimestampUs(); et = taosGetTimestampUs();
fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6); fprintf(stderr, "\nDB: error: %s (%.6fs)\n",
ws_errstr(res), (et - st)/1E6);
ws_free_result(res); ws_free_result(res);
return; return;
} }
if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { if (code == TSDB_CODE_WS_SEND_TIMEOUT
|| code == TSDB_CODE_WS_RECV_TIMEOUT) {
fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n");
} else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { } else if (code == TSDB_CODE_WS_INTERNAL_ERRO
|| code == TSDB_CODE_WS_CLOSED) {
shell.ws_conn = NULL; shell.ws_conn = NULL;
} }
ws_free_result(res); ws_free_result(res);
...@@ -252,7 +288,8 @@ void shellRunSingleCommandWebsocketImp(char *command) { ...@@ -252,7 +288,8 @@ void shellRunSingleCommandWebsocketImp(char *command) {
execute_time = ws_take_timing(res)/1E6; execute_time = ws_take_timing(res)/1E6;
} }
if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$",
REG_EXTENDED | REG_ICASE)) {
fprintf(stdout, "Database changed.\r\n\r\n"); fprintf(stdout, "Database changed.\r\n\r\n");
fflush(stdout); fflush(stdout);
ws_free_result(res); ws_free_result(res);
...@@ -266,10 +303,12 @@ void shellRunSingleCommandWebsocketImp(char *command) { ...@@ -266,10 +303,12 @@ void shellRunSingleCommandWebsocketImp(char *command) {
double total_time = (et - st)/1E3; double total_time = (et - st)/1E3;
double net_time = total_time - (double)execute_time; double net_time = total_time - (double)execute_time;
printf("Query Ok, %d of %d row(s) in database\n", numOfRows, numOfRows); printf("Query Ok, %d of %d row(s) in database\n", numOfRows, numOfRows);
printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", execute_time, net_time, total_time); printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n",
execute_time, net_time, total_time);
} else { } else {
int error_no = 0; int error_no = 0;
numOfRows = shellDumpWebsocket(res, fname, &error_no, printMode, &execute_time); numOfRows = shellDumpWebsocket(res, fname, &error_no,
printMode, &execute_time);
if (numOfRows < 0) { if (numOfRows < 0) {
ws_free_result(res); ws_free_result(res);
return; return;
...@@ -279,11 +318,13 @@ void shellRunSingleCommandWebsocketImp(char *command) { ...@@ -279,11 +318,13 @@ void shellRunSingleCommandWebsocketImp(char *command) {
double net_time = total_time - execute_time; double net_time = total_time - execute_time;
if (error_no == 0 && !shell.stop_query) { if (error_no == 0 && !shell.stop_query) {
printf("Query OK, %d row(s) in set\n", numOfRows); printf("Query OK, %d row(s) in set\n", numOfRows);
printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", execute_time, net_time, total_time); printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n",
execute_time, net_time, total_time);
} else { } else {
printf("Query interrupted, %d row(s) in set (%.6fs)\n", numOfRows, printf("Query interrupted, %d row(s) in set (%.6fs)\n", numOfRows,
(et - st)/1E6); (et - st)/1E6);
printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", execute_time, net_time, total_time); printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n",
execute_time, net_time, total_time);
} }
} }
printf("\n"); printf("\n");
......
utils/test/c/sml_test.c
浏览文件 @ 6811a6b2
...@@ -245,8 +245,7 @@ int smlProcess_json3_Test() { ...@@ -245,8 +245,7 @@ int smlProcess_json3_Test() {
taos_free_result(pRes); taos_free_result(pRes);
const char *sql[] = { const char *sql[] = {
"[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\"," "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]"};
"\"dc\":\"lga\"}}]"};
char *sql1[1] = {0}; char *sql1[1] = {0};
for (int i = 0; i < 1; i++) { for (int i = 0; i < 1; i++) {
sql1[i] = taosMemoryCalloc(1, 1024); sql1[i] = taosMemoryCalloc(1, 1024);
......
utils/test/c/tmqSim.c
浏览文件 @ 6811a6b2
...@@ -688,16 +688,17 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { ...@@ -688,16 +688,17 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) {
} }
static int32_t g_once_commit_flag = 0; static int32_t g_once_commit_flag = 0;
static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code);
if (0 == g_once_commit_flag) { static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
g_once_commit_flag = 1; taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code);
notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT);
if (0 == g_once_commit_flag) {
g_once_commit_flag = 1;
notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT);
} }
char tmpString[128]; char tmpString[128];
taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString));
} }
void build_consumer(SThreadInfo* pInfo) { void build_consumer(SThreadInfo* pInfo) {
...@@ -858,7 +859,9 @@ void loop_consume(SThreadInfo* pInfo) { ...@@ -858,7 +859,9 @@ void loop_consume(SThreadInfo* pInfo) {
taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n", taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n",
pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt);
taosFsyncFile(pInfo->pConsumeRowsFile); if(taosFsyncFile(pInfo->pConsumeRowsFile) < 0){
printf("taosFsyncFile error:%s", strerror(errno));
}
taosCloseFile(&pInfo->pConsumeRowsFile); taosCloseFile(&pInfo->pConsumeRowsFile);
} }
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或