Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
gjl2004yn
jumpserver
提交
b8f175e4
J
jumpserver
项目概览
gjl2004yn
/
jumpserver
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
J
jumpserver
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
b8f175e4
编写于
11月 20, 2020
作者:
F
fit2bot
提交者:
GitHub
11月 20, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
perf(celery-task): 优化检查节点资产数量的 Celery 任务 (#5052)
Co-authored-by:
N
xinwen
<
coderWen@126.com
>
上级
a626ff5a
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
26 addition
and
23 deletion
+26
-23
apps/assets/tasks/nodes_amount.py
apps/assets/tasks/nodes_amount.py
+3
-4
apps/assets/utils.py
apps/assets/utils.py
+7
-2
apps/ops/celery/__init__.py
apps/ops/celery/__init__.py
+0
-13
apps/perms/tasks.py
apps/perms/tasks.py
+4
-1
jms
jms
+12
-3
未找到文件。
apps/assets/tasks/nodes_amount.py
浏览文件 @
b8f175e4
from
celery
import
shared_task
from
ops.celery.decorator
import
register_as_period_task
from
assets.utils
import
check_node_assets_amount
from
common.utils
import
get_logger
from
common.utils.timezone
import
now
logger
=
get_logger
(
__file__
)
@
shared_task
()
@
register_as_period_task
(
crontab
=
'* 2 * * *'
)
@
shared_task
(
queue
=
'celery_heavy_tasks'
)
def
check_node_assets_amount_celery_task
():
logger
.
info
(
f
'>>>
{
now
()
}
begin check_node_assets_amount_celery_task ...'
)
check_node_assets_amount
()
logger
.
info
(
f
'>>>
{
now
()
}
end check_node_assets_amount_celery_task ...'
)
apps/assets/utils.py
浏览文件 @
b8f175e4
# ~*~ coding: utf-8 ~*~
#
import
time
from
django.db.models
import
Q
from
common.utils
import
get_logger
,
dict_get_any
,
is_uuid
,
get_object_or_none
...
...
@@ -12,15 +14,18 @@ logger = get_logger(__file__)
def
check_node_assets_amount
():
for
node
in
Node
.
objects
.
all
():
logger
.
info
(
f
'Check node assets amount:
{
node
}
'
)
assets_amount
=
Asset
.
objects
.
filter
(
Q
(
nodes__key__istartswith
=
f
'
{
node
.
key
}
:'
)
|
Q
(
nodes
=
node
)
).
distinct
().
count
()
if
node
.
assets_amount
!=
assets_amount
:
print
(
f
'>>> <Node:
{
node
.
key
}
> wrong assets amount
'
f
'
{
node
.
assets_amount
}
right is
{
assets_amount
}
'
)
logger
.
warn
(
f
'Node wrong assets amount <Node:
{
node
.
key
}
>
'
f
'
{
node
.
assets_amount
}
right is
{
assets_amount
}
'
)
node
.
assets_amount
=
assets_amount
node
.
save
()
# 防止自检程序给数据库的压力太大
time
.
sleep
(
0.1
)
def
is_asset_exists_in_node
(
asset_pk
,
node_key
):
...
...
apps/ops/celery/__init__.py
浏览文件 @
b8f175e4
...
...
@@ -29,16 +29,3 @@ configs["CELERY_ROUTES"] = {
app
.
namespace
=
'CELERY'
app
.
conf
.
update
(
configs
)
app
.
autodiscover_tasks
(
lambda
:
[
app_config
.
split
(
'.'
)[
0
]
for
app_config
in
settings
.
INSTALLED_APPS
])
app
.
conf
.
beat_schedule
=
{
'check-asset-permission-expired'
:
{
'task'
:
'perms.tasks.check_asset_permission_expired'
,
'schedule'
:
settings
.
PERM_EXPIRED_CHECK_PERIODIC
,
'args'
:
()
},
'check-node-assets-amount'
:
{
'task'
:
'assets.tasks.nodes_amount.check_node_assets_amount_celery_task'
,
'schedule'
:
crontab
(
minute
=
0
,
hour
=
0
),
'args'
:
()
},
}
apps/perms/tasks.py
浏览文件 @
b8f175e4
...
...
@@ -5,10 +5,12 @@ from datetime import timedelta
from
django.db
import
transaction
from
django.db.models
import
Q
from
django.db.transaction
import
atomic
from
django.conf
import
settings
from
celery
import
shared_task
from
common.utils
import
get_logger
from
common.utils.timezone
import
now
,
dt_formater
,
dt_parser
from
users.models
import
User
from
ops.celery.decorator
import
register_as_period_task
from
assets.models
import
Node
from
perms.models
import
RebuildUserTreeTask
,
AssetPermission
from
perms.utils.asset.user_permission
import
rebuild_user_mapping_nodes_if_need_with_lock
,
lock
...
...
@@ -33,7 +35,8 @@ def dispatch_mapping_node_tasks():
rebuild_user_mapping_nodes_celery_task
.
delay
(
id
)
@
shared_task
(
queue
=
'check_asset_perm_expired'
)
@
register_as_period_task
(
interval
=
settings
.
PERM_EXPIRED_CHECK_PERIODIC
)
@
shared_task
(
queue
=
'celery_check_asset_perm_expired'
)
@
atomic
()
def
check_asset_permission_expired
():
"""
...
...
jms
浏览文件 @
b8f175e4
...
...
@@ -156,7 +156,10 @@ def is_running(s, unlink=True):
def
parse_service
(
s
):
web_services
=
[
'gunicorn'
,
'flower'
,
'daphne'
]
celery_services
=
[
"celery_ansible"
,
"celery_default"
,
"celery_node_tree"
,
"check_asset_perm_expired"
]
celery_services
=
[
"celery_ansible"
,
"celery_default"
,
"celery_node_tree"
,
"celery_check_asset_perm_expired"
,
"celery_heavy_tasks"
]
task_services
=
celery_services
+
[
'beat'
]
all_services
=
web_services
+
task_services
if
s
==
'all'
:
...
...
@@ -225,9 +228,14 @@ def get_start_celery_node_tree_kwargs():
return
get_start_worker_kwargs
(
'node_tree'
,
2
)
def
get_start_celery_heavy_tasks_kwargs
():
print
(
"
\n
- Start Celery as Distributed Task Queue: HeavyTasks"
)
return
get_start_worker_kwargs
(
'celery_heavy_tasks'
,
1
)
def
get_start_celery_check_asset_perm_expired_kwargs
():
print
(
"
\n
- Start Celery as Distributed Task Queue: CheckAseetPermissionExpired"
)
return
get_start_worker_kwargs
(
'check_asset_perm_expired'
,
1
)
return
get_start_worker_kwargs
(
'c
elery_c
heck_asset_perm_expired'
,
1
)
def
get_start_worker_kwargs
(
queue
,
num
):
...
...
@@ -366,7 +374,8 @@ def start_service(s):
"celery_ansible"
:
get_start_celery_ansible_kwargs
,
"celery_default"
:
get_start_celery_default_kwargs
,
"celery_node_tree"
:
get_start_celery_node_tree_kwargs
,
"check_asset_perm_expired"
:
get_start_celery_check_asset_perm_expired_kwargs
,
"celery_heavy_tasks"
:
get_start_celery_heavy_tasks_kwargs
,
"celery_check_asset_perm_expired"
:
get_start_celery_check_asset_perm_expired_kwargs
,
"beat"
:
get_start_beat_kwargs
,
"flower"
:
get_start_flower_kwargs
,
"daphne"
:
get_start_daphne_kwargs
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录