Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
a630d82c
T
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1180
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
a630d82c
编写于
12月 22, 2021
作者:
S
Shengliang Guan
提交者:
GitHub
12月 22, 2021
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #9262 from taosdata/feature/qnode
Feature/qnode
上级
26a3ff9b
7a622fa6
变更
14
隐藏空白更改
内联
并排
Showing
14 changed file
with
523 addition
and
85 deletion
+523
-85
include/common/taosmsg.h
include/common/taosmsg.h
+26
-1
include/dnode/qnode/qnode.h
include/dnode/qnode/qnode.h
+1
-43
include/libs/qcom/query.h
include/libs/qcom/query.h
+9
-0
include/libs/qworker/qworker.h
include/libs/qworker/qworker.h
+62
-0
include/libs/scheduler/scheduler.h
include/libs/scheduler/scheduler.h
+1
-0
include/util/thash.h
include/util/thash.h
+1
-2
source/libs/CMakeLists.txt
source/libs/CMakeLists.txt
+1
-0
source/libs/executor/src/tfilter.c
source/libs/executor/src/tfilter.c
+1
-1
source/libs/qworker/CMakeLists.txt
source/libs/qworker/CMakeLists.txt
+12
-0
source/libs/qworker/inc/qworkerInt.h
source/libs/qworker/inc/qworkerInt.h
+68
-0
source/libs/qworker/src/qworker.c
source/libs/qworker/src/qworker.c
+298
-0
source/libs/scheduler/inc/schedulerInt.h
source/libs/scheduler/inc/schedulerInt.h
+2
-10
source/libs/scheduler/src/scheduler.c
source/libs/scheduler/src/scheduler.c
+32
-26
source/util/src/thash.c
source/util/src/thash.c
+9
-2
未找到文件。
include/common/taosmsg.h
浏览文件 @
a630d82c
...
...
@@ -51,7 +51,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" )
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_MQ_CONNECT
,
"mq-connect"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_MQ_DISCONNECT
,
"mq-disconnect"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_MQ_SET_CUR
,
"mq-set-cur"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_R
SP_READY
,
"rsp
-ready"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_R
ES_READY
,
"res
-ready"
)
// message from client to mnode
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_CONNECT
,
"connect"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_CREATE_ACCT
,
"create-acct"
)
...
...
@@ -1094,6 +1094,7 @@ typedef struct {
}
SUpdateTagValRsp
;
typedef
struct
SSchedulerQueryMsg
{
uint64_t
schedulerId
;
uint64_t
queryId
;
uint64_t
taskId
;
uint32_t
contentLen
;
...
...
@@ -1101,15 +1102,39 @@ typedef struct SSchedulerQueryMsg {
}
SSchedulerQueryMsg
;
typedef
struct
SSchedulerReadyMsg
{
uint64_t
schedulerId
;
uint64_t
queryId
;
uint64_t
taskId
;
}
SSchedulerReadyMsg
;
typedef
struct
SSchedulerFetchMsg
{
uint64_t
schedulerId
;
uint64_t
queryId
;
uint64_t
taskId
;
}
SSchedulerFetchMsg
;
typedef
struct
SSchedulerStatusMsg
{
uint64_t
schedulerId
;
}
SSchedulerStatusMsg
;
typedef
struct
STaskStatus
{
uint64_t
queryId
;
uint64_t
taskId
;
int8_t
status
;
}
STaskStatus
;
typedef
struct
SSchedulerStatusRsp
{
uint32_t
num
;
STaskStatus
status
[];
}
SSchedulerStatusRsp
;
typedef
struct
SSchedulerCancelMsg
{
uint64_t
schedulerId
;
uint64_t
queryId
;
uint64_t
taskId
;
}
SSchedulerCancelMsg
;
#pragma pack(pop)
...
...
include/dnode/qnode/qnode.h
浏览文件 @
a630d82c
...
...
@@ -19,7 +19,7 @@
#ifdef __cplusplus
extern
"C"
{
#endif
#include "trpc.h"
typedef
struct
{
uint64_t
numOfStartTask
;
...
...
@@ -32,48 +32,6 @@ typedef struct {
uint64_t
numOfErrors
;
}
SQnodeStat
;
/* start Task msg */
typedef
struct
{
uint32_t
schedulerIp
;
uint16_t
schedulerPort
;
int64_t
taskId
;
int64_t
queryId
;
uint32_t
srcIp
;
uint16_t
srcPort
;
}
SQnodeStartTaskMsg
;
/* stop Task msg */
typedef
struct
{
int64_t
taskId
;
}
SQnodeStopTaskMsg
;
/* start/stop Task msg response */
typedef
struct
{
int64_t
taskId
;
int32_t
code
;
}
SQnodeTaskRespMsg
;
/* Task status msg */
typedef
struct
{
int64_t
taskId
;
int32_t
status
;
int64_t
queryId
;
}
SQnodeTaskStatusMsg
;
/* Qnode/Scheduler heartbeat msg */
typedef
struct
{
int32_t
status
;
int32_t
load
;
}
SQnodeHeartbeatMsg
;
/* Qnode sent/received msg */
typedef
struct
{
int8_t
msgType
;
int32_t
msgLen
;
char
msg
[];
}
SQnodeMsg
;
/**
* Start one Qnode in Dnode.
...
...
include/libs/qcom/query.h
浏览文件 @
a630d82c
...
...
@@ -24,6 +24,15 @@ extern "C" {
#include "thash.h"
#include "tlog.h"
enum
{
JOB_TASK_STATUS_NOT_START
=
1
,
JOB_TASK_STATUS_EXECUTING
,
JOB_TASK_STATUS_SUCCEED
,
JOB_TASK_STATUS_FAILED
,
JOB_TASK_STATUS_CANCELLING
,
JOB_TASK_STATUS_CANCELLED
};
typedef
struct
STableComInfo
{
uint8_t
numOfTags
;
// the number of tags in schema
uint8_t
precision
;
// the number of precision
...
...
include/libs/qworker/qworker.h
0 → 100644
浏览文件 @
a630d82c
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_QWORKER_H_
#define _TD_QWORKER_H_
#ifdef __cplusplus
extern
"C"
{
#endif
#include "trpc.h"
typedef
struct
SQWorkerCfg
{
uint32_t
maxSchedulerNum
;
uint32_t
maxResCacheNum
;
uint32_t
maxSchTaskNum
;
}
SQWorkerCfg
;
typedef
struct
{
uint64_t
numOfStartTask
;
uint64_t
numOfStopTask
;
uint64_t
numOfRecvedFetch
;
uint64_t
numOfSentHb
;
uint64_t
numOfSentFetch
;
uint64_t
numOfTaskInQueue
;
uint64_t
numOfFetchInQueue
;
uint64_t
numOfErrors
;
}
SQWorkerStat
;
int32_t
qWorkerInit
(
SQWorkerCfg
*
cfg
,
void
**
qWorkerMgmt
);
int32_t
qWorkerProcessQueryMsg
(
void
*
qWorkerMgmt
,
SSchedulerQueryMsg
*
msg
,
SRpcMsg
*
rsp
);
int32_t
qWorkerProcessReadyMsg
(
void
*
qWorkerMgmt
,
SSchedulerReadyMsg
*
msg
,
SRpcMsg
*
rsp
);
int32_t
qWorkerProcessStatusMsg
(
void
*
qWorkerMgmt
,
SSchedulerStatusMsg
*
msg
,
SRpcMsg
*
rsp
);
int32_t
qWorkerProcessFetchMsg
(
void
*
qWorkerMgmt
,
SSchedulerFetchMsg
*
msg
,
SRpcMsg
*
rsp
);
int32_t
qWorkerProcessCancelMsg
(
void
*
qWorkerMgmt
,
SSchedulerCancelMsg
*
msg
,
SRpcMsg
*
rsp
);
void
qWorkerDestroy
(
void
**
qWorkerMgmt
);
#ifdef __cplusplus
}
#endif
#endif
/*_TD_QWORKER_H_*/
include/libs/scheduler/scheduler.h
浏览文件 @
a630d82c
...
...
@@ -25,6 +25,7 @@ extern "C" {
typedef
struct
SSchedulerCfg
{
int32_t
clusterType
;
int32_t
maxJobNum
;
}
SSchedulerCfg
;
typedef
struct
SQueryProfileSummary
{
...
...
include/util/thash.h
浏览文件 @
a630d82c
...
...
@@ -193,11 +193,10 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p);
/**
* Get the corresponding key information for a given data in hash table
* @param pHashObj
* @param data
* @return
*/
int32_t
taosHashGetKey
(
SHashObj
*
pHashObj
,
void
*
data
,
void
**
key
,
size_t
*
keyLen
);
int32_t
taosHashGetKey
(
void
*
data
,
void
**
key
,
size_t
*
keyLen
);
#ifdef __cplusplus
}
...
...
source/libs/CMakeLists.txt
浏览文件 @
a630d82c
...
...
@@ -11,3 +11,4 @@ add_subdirectory(executor)
add_subdirectory
(
planner
)
add_subdirectory
(
function
)
add_subdirectory
(
qcom
)
add_subdirectory
(
qworker
)
source/libs/executor/src/tfilter.c
浏览文件 @
a630d82c
...
...
@@ -1175,7 +1175,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
void
*
key
=
NULL
;
len
=
0
;
taosHashGetKey
(
(
SHashObj
*
)
data
,
p
,
&
key
,
&
len
);
taosHashGetKey
(
p
,
&
key
,
&
len
);
void
*
fdata
=
NULL
;
if
(
IS_VAR_DATA_TYPE
(
type
))
{
...
...
source/libs/qworker/CMakeLists.txt
0 → 100644
浏览文件 @
a630d82c
aux_source_directory
(
src QWORKER_SRC
)
add_library
(
qworker
${
QWORKER_SRC
}
)
target_include_directories
(
qworker
PUBLIC
"
${
CMAKE_SOURCE_DIR
}
/include/libs/qworker"
PRIVATE
"
${
CMAKE_CURRENT_SOURCE_DIR
}
/inc"
)
target_link_libraries
(
qworker
PRIVATE os util transport planner qcom
)
source/libs/qworker/inc/qworkerInt.h
0 → 100644
浏览文件 @
a630d82c
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_QWORKER_INT_H_
#define _TD_QWORKER_INT_H_
#ifdef __cplusplus
extern
"C"
{
#endif
#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000
#define QWORKER_DEFAULT_RES_CACHE_NUMBER 10000
#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000
enum
{
QW_READY_NOT_RECEIVED
=
0
,
QW_READY_RECEIVED
,
QW_READY_RESPONSED
,
};
typedef
struct
SQWorkerTaskStatus
{
int8_t
status
;
int8_t
ready
;
}
SQWorkerTaskStatus
;
typedef
struct
SQWorkerResCache
{
void
*
data
;
}
SQWorkerResCache
;
typedef
struct
SQWorkerSchTaskStatus
{
int32_t
lastAccessTs
;
// timestamp in second
SHashObj
*
taskStatus
;
// key:queryId+taskId, value: SQWorkerTaskStatus
}
SQWorkerSchTaskStatus
;
// Qnode/Vnode level task management
typedef
struct
SQWorkerMgmt
{
SQWorkerCfg
cfg
;
SHashObj
*
scheduleHash
;
//key: schedulerId, value: SQWorkerSchTaskStatus
SHashObj
*
resHash
;
//key: queryId+taskId, value: SQWorkerResCache
}
SQWorkerMgmt
;
#define QW_TASK_DONE(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == status == JOB_TASK_STATUS_CANCELLED)
#define QW_SET_QTID(id, qid, tid) do { *(uint64_t *)(id) = (qid); *(uint64_t *)((char *)(id) + sizeof(qid)) = (tid); } while (0)
#define QW_GET_QTID(id, qid, tid) do { (qid) = *(uint64_t *)(id); (tid) = *(uint64_t *)((char *)(id) + sizeof(qid)); } while (0)
#define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0)
#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#ifdef __cplusplus
}
#endif
#endif
/*_TD_QWORKER_INT_H_*/
source/libs/qworker/src/qworker.c
0 → 100644
浏览文件 @
a630d82c
#include "taosmsg.h"
#include "query.h"
#include "qworker.h"
#include "qworkerInt.h"
#include "planner.h"
int32_t
qwAddTaskStatus
(
SQWorkerMgmt
*
mgmt
,
uint64_t
schedulerId
,
uint64_t
queryId
,
uint64_t
taskId
,
int8_t
taskStatus
)
{
SQWorkerTaskStatus
tStatus
=
{
0
};
tStatus
.
status
=
taskStatus
;
char
id
[
sizeof
(
queryId
)
+
sizeof
(
taskId
)]
=
{
0
};
QW_SET_QTID
(
id
,
queryId
,
taskId
);
SQWorkerSchTaskStatus
*
schStatus
=
taosHashGet
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
));
if
(
NULL
==
schStatus
)
{
SQWorkerSchTaskStatus
newSchStatus
=
{
0
};
newSchStatus
.
taskStatus
=
taosHashInit
(
mgmt
->
cfg
.
maxSchTaskNum
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
newSchStatus
.
taskStatus
)
{
qError
(
"taosHashInit %d failed"
,
mgmt
->
cfg
.
maxSchTaskNum
);
return
TSDB_CODE_QRY_OUT_OF_MEMORY
;
}
if
(
0
!=
taosHashPut
(
newSchStatus
.
taskStatus
,
id
,
sizeof
(
id
),
&
tStatus
,
sizeof
(
tStatus
)))
{
qError
(
"taosHashPut schedulerId[%"
PRIx64
"]queryId[%"
PRIx64
"] taskId[%"
PRIx64
"] to scheduleHash failed"
,
schedulerId
,
queryId
,
taskId
);
taosHashCleanup
(
newSchStatus
.
taskStatus
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
newSchStatus
.
lastAccessTs
=
taosGetTimestampSec
();
if
(
0
!=
taosHashPut
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
),
&
newSchStatus
,
sizeof
(
newSchStatus
)))
{
qError
(
"taosHashPut schedulerId[%"
PRIx64
"] to scheduleHash failed"
,
schedulerId
);
taosHashCleanup
(
newSchStatus
.
taskStatus
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
return
TSDB_CODE_SUCCESS
;
}
schStatus
->
lastAccessTs
=
taosGetTimestampSec
();
if
(
0
!=
taosHashPut
(
schStatus
->
taskStatus
,
id
,
sizeof
(
id
),
&
tStatus
,
sizeof
(
tStatus
)))
{
qError
(
"taosHashPut schedulerId[%"
PRIx64
"]queryId[%"
PRIx64
"] taskId[%"
PRIx64
"] to scheduleHash failed"
,
schedulerId
,
queryId
,
taskId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwGetTaskStatus
(
SQWorkerMgmt
*
mgmt
,
uint64_t
schedulerId
,
uint64_t
queryId
,
uint64_t
taskId
,
SQWorkerTaskStatus
**
taskStatus
)
{
SQWorkerSchTaskStatus
*
schStatus
=
taosHashGet
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
));
if
(
NULL
==
schStatus
)
{
qError
(
"no scheduler for schedulerId[%"
PRIx64
"]"
,
schedulerId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
schStatus
->
lastAccessTs
=
taosGetTimestampSec
();
char
id
[
sizeof
(
queryId
)
+
sizeof
(
taskId
)]
=
{
0
};
QW_SET_QTID
(
id
,
queryId
,
taskId
);
SQWorkerTaskStatus
*
tStatus
=
taosHashGet
(
schStatus
->
taskStatus
,
id
,
sizeof
(
id
));
if
(
NULL
==
tStatus
)
{
qError
(
"no task status for schedulerId[%"
PRIx64
"] queryId[%"
PRIx64
"] taskId[%"
PRIx64
"]"
,
schedulerId
,
queryId
,
taskId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
*
taskStatus
=
tStatus
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwAddTaskResult
(
SQWorkerMgmt
*
mgmt
,
uint64_t
queryId
,
uint64_t
taskId
,
void
*
data
)
{
char
id
[
sizeof
(
queryId
)
+
sizeof
(
taskId
)]
=
{
0
};
QW_SET_QTID
(
id
,
queryId
,
taskId
);
SQWorkerResCache
resCache
=
{
0
};
resCache
.
data
=
data
;
if
(
0
!=
taosHashPut
(
mgmt
->
resHash
,
id
,
sizeof
(
id
),
&
resCache
,
sizeof
(
SQWorkerResCache
)))
{
qError
(
"taosHashPut queryId[%"
PRIx64
"] taskId[%"
PRIx64
"] to resHash failed"
,
queryId
,
taskId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwGetTaskResult
(
SQWorkerMgmt
*
mgmt
,
uint64_t
queryId
,
uint64_t
taskId
,
void
**
data
)
{
char
id
[
sizeof
(
queryId
)
+
sizeof
(
taskId
)]
=
{
0
};
QW_SET_QTID
(
id
,
queryId
,
taskId
);
SQWorkerResCache
*
resCache
=
taosHashGet
(
mgmt
->
resHash
,
id
,
sizeof
(
id
));
if
(
NULL
==
resCache
)
{
qError
(
"no task res for queryId[%"
PRIx64
"] taskId[%"
PRIx64
"]"
,
queryId
,
taskId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
*
data
=
resCache
->
data
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwUpdateSchLastAccess
(
SQWorkerMgmt
*
mgmt
,
uint64_t
schedulerId
)
{
SQWorkerSchTaskStatus
*
schStatus
=
taosHashGet
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
));
if
(
NULL
==
schStatus
)
{
qError
(
"no scheduler for schedulerId[%"
PRIx64
"]"
,
schedulerId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
schStatus
->
lastAccessTs
=
taosGetTimestampSec
();
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwGetSchTasksStatus
(
SQWorkerMgmt
*
mgmt
,
uint64_t
schedulerId
,
SSchedulerStatusRsp
**
rsp
)
{
SQWorkerSchTaskStatus
*
schStatus
=
taosHashGet
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
));
if
(
NULL
==
schStatus
)
{
qError
(
"no scheduler for schedulerId[%"
PRIx64
"]"
,
schedulerId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
schStatus
->
lastAccessTs
=
taosGetTimestampSec
();
int32_t
i
=
0
;
int32_t
taskNum
=
taosHashGetSize
(
schStatus
->
taskStatus
);
int32_t
size
=
sizeof
(
SSchedulerStatusRsp
)
+
sizeof
((
*
rsp
)
->
status
[
0
])
*
taskNum
;
*
rsp
=
calloc
(
1
,
size
);
if
(
NULL
==
*
rsp
)
{
qError
(
"calloc %d failed"
,
size
);
return
TSDB_CODE_QRY_OUT_OF_MEMORY
;
}
void
*
key
=
NULL
;
size_t
keyLen
=
0
;
void
*
pIter
=
taosHashIterate
(
schStatus
->
taskStatus
,
NULL
);
while
(
pIter
)
{
SQWorkerTaskStatus
*
taskStatus
=
(
SQWorkerTaskStatus
*
)
pIter
;
taosHashGetKey
(
pIter
,
&
key
,
&
keyLen
);
QW_GET_QTID
(
key
,
(
*
rsp
)
->
status
[
i
].
queryId
,
(
*
rsp
)
->
status
[
i
].
taskId
);
(
*
rsp
)
->
status
[
i
].
status
=
taskStatus
->
status
;
pIter
=
taosHashIterate
(
schStatus
->
taskStatus
,
pIter
);
}
(
*
rsp
)
->
num
=
taskNum
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwBuildRspMsg
(
void
*
data
,
int32_t
msgType
);
int32_t
qWorkerInit
(
SQWorkerCfg
*
cfg
,
void
**
qWorkerMgmt
)
{
SQWorkerMgmt
*
mgmt
=
calloc
(
1
,
sizeof
(
SQWorkerMgmt
));
if
(
NULL
==
mgmt
)
{
qError
(
"calloc %d failed"
,
(
int32_t
)
sizeof
(
SQWorkerMgmt
));
return
TSDB_CODE_QRY_OUT_OF_MEMORY
;
}
if
(
cfg
)
{
mgmt
->
cfg
=
*
cfg
;
}
else
{
mgmt
->
cfg
.
maxSchedulerNum
=
QWORKER_DEFAULT_SCHEDULER_NUMBER
;
mgmt
->
cfg
.
maxResCacheNum
=
QWORKER_DEFAULT_RES_CACHE_NUMBER
;
mgmt
->
cfg
.
maxSchTaskNum
=
QWORKER_DEFAULT_SCH_TASK_NUMBER
;
}
mgmt
->
scheduleHash
=
taosHashInit
(
mgmt
->
cfg
.
maxSchedulerNum
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_UBIGINT
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
mgmt
->
scheduleHash
)
{
tfree
(
mgmt
);
QW_ERR_LRET
(
TSDB_CODE_QRY_OUT_OF_MEMORY
,
"init %d schduler hash failed"
,
mgmt
->
cfg
.
maxSchedulerNum
);
}
mgmt
->
resHash
=
taosHashInit
(
mgmt
->
cfg
.
maxResCacheNum
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
mgmt
->
resHash
)
{
taosHashCleanup
(
mgmt
->
scheduleHash
);
mgmt
->
scheduleHash
=
NULL
;
tfree
(
mgmt
);
QW_ERR_LRET
(
TSDB_CODE_QRY_OUT_OF_MEMORY
,
"init %d res cache hash failed"
,
mgmt
->
cfg
.
maxResCacheNum
);
}
*
qWorkerMgmt
=
mgmt
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qWorkerProcessQueryMsg
(
void
*
qWorkerMgmt
,
SSchedulerQueryMsg
*
msg
,
SRpcMsg
*
rsp
)
{
if
(
NULL
==
qWorkerMgmt
||
NULL
==
msg
||
NULL
==
rsp
)
{
return
TSDB_CODE_QRY_INVALID_INPUT
;
}
SSubplan
*
plan
=
NULL
;
SQWorkerTaskStatus
*
tStatus
=
NULL
;
int32_t
code
=
qStringToSubplan
(
msg
->
msg
,
&
plan
);
if
(
TSDB_CODE_SUCCESS
!=
code
)
{
qError
(
"schId:%"
PRIx64
",qId:%"
PRIx64
",taskId:%"
PRIx64
" string to subplan failed, code:%d"
,
msg
->
schedulerId
,
msg
->
queryId
,
msg
->
taskId
,
code
);
return
code
;
}
//TODO call executer to init subquery
QW_ERR_JRET
(
qwAddTaskStatus
(
qWorkerMgmt
,
msg
->
schedulerId
,
msg
->
queryId
,
msg
->
taskId
,
JOB_TASK_STATUS_EXECUTING
));
QW_ERR_JRET
(
qwBuildRspMsg
(
NULL
,
TSDB_MSG_TYPE_QUERY_RSP
));
//TODO call executer to execute subquery
code
=
0
;
void
*
data
=
NULL
;
//TODO call executer to execute subquery
QW_ERR_JRET
(
qwGetTaskStatus
(
qWorkerMgmt
,
msg
->
schedulerId
,
msg
->
queryId
,
msg
->
taskId
,
&
tStatus
));
tStatus
->
status
=
(
code
)
?
JOB_TASK_STATUS_FAILED
:
JOB_TASK_STATUS_SUCCEED
;
QW_ERR_JRET
(
qwAddTaskResult
(
qWorkerMgmt
,
msg
->
queryId
,
msg
->
taskId
,
data
));
_return:
if
(
tStatus
&&
QW_TASK_DONE
(
tStatus
->
status
)
&&
QW_READY_RECEIVED
==
tStatus
->
ready
)
{
QW_ERR_RET
(
qwBuildRspMsg
(
NULL
,
TSDB_MSG_TYPE_RES_READY_RSP
));
}
qDestroySubplan
(
plan
);
return
code
;
}
int32_t
qWorkerProcessReadyMsg
(
void
*
qWorkerMgmt
,
SSchedulerReadyMsg
*
msg
,
SRpcMsg
*
rsp
){
if
(
NULL
==
qWorkerMgmt
||
NULL
==
msg
||
NULL
==
rsp
)
{
return
TSDB_CODE_QRY_INVALID_INPUT
;
}
SQWorkerTaskStatus
*
tStatus
=
NULL
;
QW_ERR_RET
(
qwGetTaskStatus
(
qWorkerMgmt
,
msg
->
schedulerId
,
msg
->
queryId
,
msg
->
taskId
,
&
tStatus
));
if
(
QW_TASK_DONE
(
tStatus
->
status
))
{
QW_ERR_RET
(
qwBuildRspMsg
(
tStatus
,
TSDB_MSG_TYPE_RES_READY_RSP
));
}
else
{
tStatus
->
ready
=
QW_READY_RECEIVED
;
return
TSDB_CODE_SUCCESS
;
}
tStatus
->
ready
=
QW_READY_RESPONSED
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qWorkerProcessStatusMsg
(
void
*
qWorkerMgmt
,
SSchedulerStatusMsg
*
msg
,
SRpcMsg
*
rsp
)
{
if
(
NULL
==
qWorkerMgmt
||
NULL
==
msg
||
NULL
==
rsp
)
{
return
TSDB_CODE_QRY_INVALID_INPUT
;
}
SSchedulerStatusRsp
*
sStatus
=
NULL
;
QW_ERR_RET
(
qwGetSchTasksStatus
(
qWorkerMgmt
,
msg
->
schedulerId
,
&
sStatus
));
return
TSDB_CODE_SUCCESS
;
}
int32_t
qWorkerProcessFetchMsg
(
void
*
qWorkerMgmt
,
SSchedulerFetchMsg
*
msg
,
SRpcMsg
*
rsp
)
{
if
(
NULL
==
qWorkerMgmt
||
NULL
==
msg
||
NULL
==
rsp
)
{
return
TSDB_CODE_QRY_INVALID_INPUT
;
}
QW_ERR_RET
(
qwUpdateSchLastAccess
(
qWorkerMgmt
,
msg
->
schedulerId
));
void
*
data
=
NULL
;
QW_ERR_RET
(
qwGetTaskResult
(
qWorkerMgmt
,
msg
->
queryId
,
msg
->
taskId
,
&
data
));
QW_ERR_RET
(
qwBuildRspMsg
(
data
,
TSDB_MSG_TYPE_FETCH_RSP
));
return
TSDB_CODE_SUCCESS
;
}
int32_t
qWorkerProcessCancelMsg
(
void
*
qWorkerMgmt
,
SSchedulerCancelMsg
*
msg
,
SRpcMsg
*
rsp
);
void
qWorkerDestroy
(
void
**
qWorkerMgmt
)
{
if
(
NULL
==
qWorkerMgmt
||
NULL
==
*
qWorkerMgmt
)
{
return
;
}
SQWorkerMgmt
*
mgmt
=
*
qWorkerMgmt
;
//TODO STOP ALL QUERY
//TODO FREE ALL
tfree
(
*
qWorkerMgmt
);
}
source/libs/scheduler/inc/schedulerInt.h
浏览文件 @
a630d82c
...
...
@@ -31,17 +31,9 @@ extern "C" {
#define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA
enum
{
SCH_STATUS_NOT_START
=
1
,
SCH_STATUS_EXECUTING
,
SCH_STATUS_SUCCEED
,
SCH_STATUS_FAILED
,
SCH_STATUS_CANCELLING
,
SCH_STATUS_CANCELLED
};
typedef
struct
SSchedulerMgmt
{
uint64_t
taskId
;
uint64_t
taskId
;
uint64_t
schedulerId
;
SSchedulerCfg
cfg
;
SHashObj
*
Jobs
;
// key: queryId, value: SQueryJob*
}
SSchedulerMgmt
;
...
...
source/libs/scheduler/src/scheduler.c
浏览文件 @
a630d82c
...
...
@@ -161,7 +161,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) {
SArray
*
levelPlans
=
NULL
;
int32_t
levelPlanNum
=
0
;
level
.
status
=
SCH
_STATUS_NOT_START
;
level
.
status
=
JOB_TASK
_STATUS_NOT_START
;
for
(
int32_t
i
=
0
;
i
<
levelNum
;
++
i
)
{
level
.
level
=
i
;
...
...
@@ -191,7 +191,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) {
task
.
taskId
=
atomic_add_fetch_64
(
&
schMgmt
.
taskId
,
1
);
task
.
plan
=
plan
;
task
.
status
=
SCH
_STATUS_NOT_START
;
task
.
status
=
JOB_TASK
_STATUS_NOT_START
;
void
*
p
=
taosArrayPush
(
level
.
subTasks
,
&
task
);
if
(
NULL
==
p
)
{
...
...
@@ -304,13 +304,15 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) {
}
SSchedulerQueryMsg
*
pMsg
=
msg
;
pMsg
->
queryId
=
job
->
queryId
;
pMsg
->
taskId
=
task
->
taskId
;
pMsg
->
contentLen
=
len
;
pMsg
->
schedulerId
=
htobe64
(
schMgmt
.
schedulerId
);
pMsg
->
queryId
=
htobe64
(
job
->
queryId
);
pMsg
->
taskId
=
htobe64
(
task
->
taskId
);
pMsg
->
contentLen
=
htonl
(
len
);
memcpy
(
pMsg
->
msg
,
task
->
msg
,
len
);
break
;
}
case
TSDB_MSG_TYPE_R
SP
_READY
:
{
case
TSDB_MSG_TYPE_R
ES
_READY
:
{
msgSize
=
sizeof
(
SSchedulerReadyMsg
);
msg
=
calloc
(
1
,
msgSize
);
if
(
NULL
==
msg
)
{
...
...
@@ -319,8 +321,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) {
}
SSchedulerReadyMsg
*
pMsg
=
msg
;
pMsg
->
queryId
=
job
->
queryId
;
pMsg
->
taskId
=
task
->
taskId
;
pMsg
->
queryId
=
htobe64
(
job
->
queryId
)
;
pMsg
->
taskId
=
htobe64
(
task
->
taskId
)
;
break
;
}
case
TSDB_MSG_TYPE_FETCH
:
{
...
...
@@ -332,8 +334,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) {
}
SSchedulerFetchMsg
*
pMsg
=
msg
;
pMsg
->
queryId
=
job
->
queryId
;
pMsg
->
taskId
=
task
->
taskId
;
pMsg
->
queryId
=
htobe64
(
job
->
queryId
)
;
pMsg
->
taskId
=
htobe64
(
task
->
taskId
)
;
break
;
}
default:
...
...
@@ -376,7 +378,7 @@ _return:
int32_t
schProcessOnJobSuccess
(
SQueryJob
*
job
)
{
job
->
status
=
SCH
_STATUS_SUCCEED
;
job
->
status
=
JOB_TASK
_STATUS_SUCCEED
;
if
(
job
->
userFetch
)
{
SCH_ERR_RET
(
schFetchFromRemote
(
job
));
...
...
@@ -386,7 +388,7 @@ int32_t schProcessOnJobSuccess(SQueryJob *job) {
}
int32_t
schProcessOnJobFailure
(
SQueryJob
*
job
)
{
job
->
status
=
SCH
_STATUS_FAILED
;
job
->
status
=
JOB_TASK
_STATUS_FAILED
;
atomic_val_compare_exchange_32
(
&
job
->
remoteFetch
,
1
,
0
);
...
...
@@ -413,7 +415,7 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) {
return
TSDB_CODE_SUCCESS
;
}
task
->
status
=
SCH
_STATUS_SUCCEED
;
task
->
status
=
JOB_TASK
_STATUS_SUCCEED
;
int32_t
parentNum
=
(
int32_t
)
taosArrayGetSize
(
task
->
parents
);
if
(
parentNum
==
0
)
{
...
...
@@ -459,7 +461,7 @@ int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCod
if
(
!
needRetry
)
{
SCH_TASK_ERR_LOG
(
"task failed[%x], no more retry"
,
errCode
);
job
->
status
=
SCH
_STATUS_FAILED
;
job
->
status
=
JOB_TASK
_STATUS_FAILED
;
SCH_ERR_RET
(
schProcessOnJobFailure
(
job
));
return
TSDB_CODE_SUCCESS
;
...
...
@@ -478,13 +480,13 @@ int32_t schHandleRspMsg(SQueryJob *job, SQueryTask *task, int32_t msgType, int32
if
(
rspCode
!=
TSDB_CODE_SUCCESS
)
{
SCH_ERR_JRET
(
schProcessOnTaskFailure
(
job
,
task
,
rspCode
));
}
else
{
code
=
schAsyncSendMsg
(
job
,
task
,
TSDB_MSG_TYPE_R
SP
_READY
);
code
=
schAsyncSendMsg
(
job
,
task
,
TSDB_MSG_TYPE_R
ES
_READY
);
if
(
code
)
{
goto
_task_error
;
}
}
break
;
case
TSDB_MSG_TYPE_R
SP
_READY
:
case
TSDB_MSG_TYPE_R
ES
_READY
:
if
(
rspCode
!=
TSDB_CODE_SUCCESS
)
{
SCH_ERR_JRET
(
schProcessOnTaskFailure
(
job
,
task
,
rspCode
));
}
else
{
...
...
@@ -534,7 +536,7 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) {
SCH_ERR_RET
(
schPushTaskToExecList
(
job
,
task
));
task
->
status
=
SCH
_STATUS_EXECUTING
;
task
->
status
=
JOB_TASK
_STATUS_EXECUTING
;
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -546,22 +548,26 @@ int32_t schLaunchJob(SQueryJob *job) {
SCH_ERR_RET
(
schLaunchTask
(
job
,
task
));
}
job
->
status
=
SCH
_STATUS_EXECUTING
;
job
->
status
=
JOB_TASK
_STATUS_EXECUTING
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
schedulerInit
(
SSchedulerCfg
*
cfg
)
{
schMgmt
.
Jobs
=
taosHashInit
(
SCHEDULE_DEFAULT_JOB_NUMBER
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_UBIGINT
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
schMgmt
.
Jobs
)
{
SCH_ERR_LRET
(
TSDB_CODE_QRY_OUT_OF_MEMORY
,
"init %d schduler jobs failed"
,
SCHEDULE_DEFAULT_JOB_NUMBER
);
}
if
(
cfg
)
{
schMgmt
.
cfg
=
*
cfg
;
}
else
{
schMgmt
.
cfg
.
maxJobNum
=
SCHEDULE_DEFAULT_JOB_NUMBER
;
}
schMgmt
.
Jobs
=
taosHashInit
(
schMgmt
.
cfg
.
maxJobNum
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_UBIGINT
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
schMgmt
.
Jobs
)
{
SCH_ERR_LRET
(
TSDB_CODE_QRY_OUT_OF_MEMORY
,
"init %d schduler jobs failed"
,
schMgmt
.
cfg
.
maxJobNum
);
}
schMgmt
.
schedulerId
=
1
;
//TODO GENERATE A UUID
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -605,7 +611,7 @@ int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, voi
SCH_ERR_JRET
(
TSDB_CODE_SCH_INTERNAL_ERROR
);
}
job
->
status
=
SCH
_STATUS_NOT_START
;
job
->
status
=
JOB_TASK
_STATUS_NOT_START
;
SCH_ERR_JRET
(
schLaunchJob
(
job
));
...
...
@@ -634,7 +640,7 @@ int32_t scheduleFetchRows(void *pJob, void **data) {
return
TSDB_CODE_QRY_APP_ERROR
;
}
if
(
job
->
status
==
SCH
_STATUS_SUCCEED
)
{
if
(
job
->
status
==
JOB_TASK
_STATUS_SUCCEED
)
{
SCH_ERR_JRET
(
schFetchFromRemote
(
job
));
}
...
...
@@ -668,7 +674,7 @@ void scheduleFreeJob(void *pJob) {
return
;
}
if
(
job
->
status
==
SCH
_STATUS_EXECUTING
)
{
if
(
job
->
status
==
JOB_TASK
_STATUS_EXECUTING
)
{
scheduleCancelJob
(
pJob
);
}
}
...
...
source/util/src/thash.c
浏览文件 @
a630d82c
...
...
@@ -776,9 +776,16 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
return
(
pHashObj
->
capacity
*
(
sizeof
(
SHashEntry
)
+
POINTER_BYTES
))
+
sizeof
(
SHashNode
)
*
taosHashGetSize
(
pHashObj
)
+
sizeof
(
SHashObj
);
}
FORCE_INLINE
void
*
taosHashGetDataKey
(
SHashObj
*
pHashObj
,
void
*
data
)
{
FORCE_INLINE
int32_t
taosHashGetKey
(
void
*
data
,
void
**
key
,
size_t
*
keyLen
)
{
if
(
NULL
==
data
||
NULL
==
key
)
{
return
-
1
;
}
SHashNode
*
node
=
GET_HASH_PNODE
(
data
);
return
GET_HASH_NODE_KEY
(
node
);
*
key
=
GET_HASH_NODE_KEY
(
node
);
*
keyLen
=
node
->
keyLen
;
return
0
;
}
FORCE_INLINE
uint32_t
taosHashGetDataKeyLen
(
SHashObj
*
pHashObj
,
void
*
data
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录