Merge pull request #3243 from jumpserver/gather_asset_user

[Feature] 收集资产用户

apps/assets/api/__init__.py

@@ -6,3 +6,4 @@ from .node import *
 from .domain import *
 from .cmd_filter import *
 from .asset_user import *
+from .gathered_user import *

apps/assets/api/admin_user.py

@@ -19,7 +19,7 @@ from rest_framework import generics
 from rest_framework.response import Response
 from orgs.mixins.api import OrgBulkModelViewSet
 
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from common.utils import get_logger
 from ..hands import IsOrgAdmin
 from ..models import AdminUser, Asset

apps/assets/api/asset.py

@@ -5,9 +5,7 @@ import random
 
 from rest_framework import generics
 from rest_framework.response import Response
-from django.utils.translation import ugettext_lazy as _
 from django.shortcuts import get_object_or_404
-from django.db.models import Q
 
 from common.utils import get_logger, get_object_or_none
 from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
@@ -16,7 +14,7 @@ from ..models import Asset, AdminUser, Node
 from .. import serializers
 from ..tasks import update_asset_hardware_info_manual, \
     test_asset_connectivity_manual
-from ..utils import LabelFilter
+from ..filters import AssetByNodeFilterBackend, LabelFilterBackend
 
 
 logger = get_logger(__file__)
@@ -27,7 +25,7 @@ __all__ = [
 ]
 
 
-class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
+class AssetViewSet(OrgBulkModelViewSet):
     """
     API endpoint that allows Asset to be viewed or edited.
     """
@@ -37,7 +35,8 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
     queryset = Asset.objects.all()
     serializer_class = serializers.AssetSerializer
     permission_classes = (IsOrgAdminOrAppUser,)
-    success_message = _("%(hostname)s was %(action)s successfully")
+    extra_filter_backends = [AssetByNodeFilterBackend, LabelFilterBackend]
+    custom_filter_fields = ['admin_user_id']
 
     def set_assets_node(self, assets):
         if not isinstance(assets, list):
@@ -54,30 +53,6 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
         assets = serializer.save()
         self.set_assets_node(assets)
 
-    def filter_node(self, queryset):
-        node_id = self.request.query_params.get("node_id")
-        if not node_id:
-            return queryset
-
-        node = get_object_or_404(Node, id=node_id)
-        show_current_asset = self.request.query_params.get("show_current_asset") in ('1', 'true')
-
-        # 当前节点是顶层节点, 并且仅显示直接资产
-        if node.is_org_root() and show_current_asset:
-            queryset = queryset.filter(
-                Q(nodes=node_id) | Q(nodes__isnull=True)
-            ).distinct()
-        # 当前节点是顶层节点，显示所有资产
-        elif node.is_org_root() and not show_current_asset:
-            return queryset
-        # 当前节点不是鼎城节点，只显示直接资产
-        elif not node.is_org_root() and show_current_asset:
-            queryset = queryset.filter(nodes=node)
-        else:
-            children = node.get_all_children(with_self=True)
-            queryset = queryset.filter(nodes__in=children).distinct()
-        return queryset
-
     def filter_admin_user_id(self, queryset):
         admin_user_id = self.request.query_params.get('admin_user_id')
         if not admin_user_id:
@@ -88,7 +63,6 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
 
     def filter_queryset(self, queryset):
         queryset = super().filter_queryset(queryset)
-        queryset = self.filter_node(queryset)
         queryset = self.filter_admin_user_id(queryset)
         return queryset
 

apps/assets/api/asset_user.py

@@ -10,7 +10,7 @@ from django.http import Http404
 
 from common.permissions import IsOrgAdminOrAppUser, NeedMFAVerify
 from common.utils import get_object_or_none, get_logger
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from ..backends import AssetUserManager
 from ..models import Asset, Node, SystemUser, AdminUser
 from .. import serializers
@@ -52,7 +52,7 @@ class AssetUserSearchBackend(filters.BaseFilterBackend):
         return _queryset
 
 
-class AssetUserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
+class AssetUserViewSet(CommonApiMixin, BulkModelViewSet):
     serializer_class = serializers.AssetUserSerializer
     permission_classes = [IsOrgAdminOrAppUser]
     http_method_names = ['get', 'post']

apps/assets/api/gathered_user.py

+# -*- coding: utf-8 -*-
+#
+
+from orgs.mixins.api import OrgModelViewSet
+from assets.models import GatheredUser
+from common.permissions import IsOrgAdmin
+
+from ..serializers import GatheredUserSerializer
+from ..filters import AssetRelatedByNodeFilterBackend
+
+
+__all__ = ['GatheredUserViewSet']
+
+
+class GatheredUserViewSet(OrgModelViewSet):
+    queryset = GatheredUser.objects.all()
+    serializer_class = GatheredUserSerializer
+    permission_classes = [IsOrgAdmin]
+    extra_filter_backends = [AssetRelatedByNodeFilterBackend]
+
+    filter_fields = ['asset', 'username', 'present']
+    search_fields = ['username', 'asset__ip', 'asset__hostname']
+
+

apps/assets/filters.py

+# -*- coding: utf-8 -*-
+#
+
+import coreapi
+from rest_framework import filters
+from django.db.models import Q
+
+from common.utils import dict_get_any, is_uuid, get_object_or_none
+from .models import Node, Label
+
+
+class AssetByNodeFilterBackend(filters.BaseFilterBackend):
+    fields = ['node', 'all']
+
+    def get_schema_fields(self, view):
+        return [
+            coreapi.Field(
+                name=field, location='query', required=False,
+                type='string', example='', description='', schema=None,
+            )
+            for field in self.fields
+        ]
+
+    @staticmethod
+    def is_query_all(request):
+        query_all_arg = request.query_params.get('all')
+        show_current_asset_arg = request.query_params.get('show_current_asset')
+
+        query_all = query_all_arg == '1'
+        if show_current_asset_arg is not None:
+            query_all = show_current_asset_arg != '1'
+        return query_all
+
+    @staticmethod
+    def get_query_node(request):
+        node_id = dict_get_any(request.query_params, ['node', 'node_id'])
+        if not node_id:
+            return None, False
+
+        if is_uuid(node_id):
+            node = get_object_or_none(Node, id=node_id)
+        else:
+            node = get_object_or_none(Node, key=node_id)
+        return node, True
+
+    @staticmethod
+    def perform_query(pattern, queryset):
+        return queryset.filter(nodes__key__regex=pattern)
+
+    def filter_queryset(self, request, queryset, view):
+        node, has_query_arg = self.get_query_node(request)
+        if not has_query_arg:
+            return queryset
+
+        if node is None:
+            return queryset.none()
+        query_all = self.is_query_all(request)
+        if query_all:
+            pattern = node.get_all_children_pattern(with_self=True)
+        else:
+            pattern = node.get_children_key_pattern(with_self=True)
+        return self.perform_query(pattern, queryset)
+
+
+class LabelFilterBackend(filters.BaseFilterBackend):
+    sep = '#'
+    query_arg = 'label'
+
+    def get_schema_fields(self, view):
+        example = self.sep.join(['os', 'linux'])
+        return [
+            coreapi.Field(
+                name=self.query_arg, location='query', required=False,
+                type='string', example=example, description=''
+            )
+        ]
+
+    def get_query_labels(self, request):
+        labels_query = request.query_params.getlist(self.query_arg)
+        if not labels_query:
+            return None
+
+        q = None
+        for kv in labels_query:
+            if self.sep not in kv:
+                continue
+            key, value = kv.strip().split(self.sep)[:2]
+            if not all([key, value]):
+                continue
+            if q:
+                q |= Q(name=key, value=value)
+            else:
+                q = Q(name=key, value=value)
+        if not q:
+            return []
+        labels = Label.objects.filter(q, is_active=True)\
+            .values_list('id', flat=True)
+        return labels
+
+    def filter_queryset(self, request, queryset, view):
+        labels = self.get_query_labels(request)
+        if labels is None:
+            return queryset
+        if len(labels) == 0:
+            return queryset.none()
+        for label in labels:
+            queryset = queryset.filter(labels=label)
+        return queryset
+
+
+class AssetRelatedByNodeFilterBackend(AssetByNodeFilterBackend):
+    @staticmethod
+    def perform_query(pattern, queryset):
+        return queryset.filter(asset__nodes__key__regex=pattern).distinct()
+

apps/assets/migrations/0039_authbook_is_active.py

+# Generated by Django 2.1.7 on 2019-09-17 12:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0038_auto_20190911_1634'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='authbook',
+            name='is_active',
+            field=models.BooleanField(default=True, verbose_name='Is active'),
+        ),
+    ]

apps/assets/migrations/0040_auto_20190917_2056.py

+# Generated by Django 2.1.7 on 2019-09-17 12:56
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0039_authbook_is_active'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='adminuser',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='authbook',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='gateway',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='systemuser',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+    ]

apps/assets/migrations/0041_gathereduser.py

+# Generated by Django 2.1.7 on 2019-09-18 04:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0040_auto_20190917_2056'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GatheredUser',
+            fields=[
+                ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('username', models.CharField(blank=True, db_index=True, max_length=32, verbose_name='Username')),
+                ('present', models.BooleanField(default=True, verbose_name='Present')),
+                ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.Asset', verbose_name='Asset')),
+            ],
+            options={'ordering': ['asset'], 'verbose_name': 'GatherUser'},
+        ),
+    ]

apps/assets/models/__init__.py

@@ -9,3 +9,4 @@ from .cmd_filter import *
 from .authbook import *
 from .utils import *
 from .authbook import *
+from .gathered_user import *

apps/assets/models/authbook.py

@@ -13,7 +13,7 @@ __all__ = ['AuthBook']
 class AuthBookQuerySet(models.QuerySet):
 
     def latest_version(self):
-        return self.filter(is_latest=True)
+        return self.filter(is_latest=True).filter(is_active=True)
 
 
 class AuthBookManager(OrgManager):
@@ -24,6 +24,7 @@ class AuthBook(AssetUser):
     asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
     is_latest = models.BooleanField(default=False, verbose_name=_('Latest version'))
     version = models.IntegerField(default=1, verbose_name=_('Version'))
+    is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
 
     objects = AuthBookManager.from_queryset(AuthBookQuerySet)()
     backend = "db"
@@ -34,25 +35,25 @@ class AuthBook(AssetUser):
     class Meta:
         verbose_name = _('AuthBook')
 
-    def _set_latest(self):
-        self._remove_pre_obj_latest()
+    def set_to_latest(self):
+        self.remove_pre_latest()
         self.is_latest = True
         self.save()
 
-    def _get_pre_obj(self):
+    def get_pre_latest(self):
         pre_obj = self.__class__.objects.filter(
             username=self.username, asset=self.asset
         ).latest_version().first()
         return pre_obj
 
-    def _remove_pre_obj_latest(self):
-        pre_obj = self._get_pre_obj()
+    def remove_pre_latest(self):
+        pre_obj = self.get_pre_latest()
         if pre_obj:
             pre_obj.is_latest = False
             pre_obj.save()
 
-    def _set_version(self):
-        pre_obj = self._get_pre_obj()
+    def set_version(self):
+        pre_obj = self.get_pre_latest()
         if pre_obj:
             self.version = pre_obj.version + 1
         else:
@@ -60,8 +61,8 @@ class AuthBook(AssetUser):
         self.save()
 
     def set_version_and_latest(self):
-        self._set_version()
-        self._set_latest()
+        self.set_version()
+        self.set_to_latest()
 
     def get_related_assets(self):
         return [self.asset]

apps/assets/models/base.py

@@ -26,7 +26,7 @@ logger = get_logger(__file__)
 class AssetUser(OrgModelMixin):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_('Name'))
-    username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric])
+    username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric], db_index=True)
     password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
     private_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH private key'))
     public_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH public key'))

apps/assets/models/gathered_user.py

+# -*- coding: utf-8 -*-
+#
+import uuid
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from orgs.mixins.models import OrgModelMixin
+
+__all__ = ['GatheredUser']
+
+
+class GatheredUser(OrgModelMixin):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_("Asset"))
+    username = models.CharField(max_length=32, blank=True, db_index=True,
+                                verbose_name=_('Username'))
+    present = models.BooleanField(default=True, verbose_name=_("Present"))
+    date_created = models.DateTimeField(auto_now_add=True,
+                                        verbose_name=_("Date created"))
+    date_updated = models.DateTimeField(auto_now=True,
+                                        verbose_name=_("Date updated"))
+
+    @property
+    def hostname(self):
+        return self.asset.hostname
+
+    @property
+    def ip(self):
+        return self.asset.ip
+
+    class Meta:
+        verbose_name = _('GatherUser')
+        ordering = ['asset']
+
+    def __str__(self):
+        return '{}: {}'.format(self.asset.hostname, self.username)
+
+
+

apps/assets/models/node.py

@@ -10,7 +10,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext
 from django.core.cache import cache
 
-from common.utils import get_logger
+from common.utils import get_logger, timeit
 from orgs.mixins.models import OrgModelMixin, OrgManager
 from orgs.utils import set_current_org, get_current_org, tmp_to_org
 from orgs.models import Organization
@@ -116,16 +116,24 @@ class FamilyMixin:
     def all_children(self):
         return self.get_all_children(with_self=False)
 
-    def get_children(self, with_self=False):
+    def get_children_key_pattern(self, with_self=False):
         pattern = r'^{0}:[0-9]+$'.format(self.key)
         if with_self:
             pattern += r'|^{0}$'.format(self.key)
+        return pattern
+
+    def get_children(self, with_self=False):
+        pattern = self.get_children_key_pattern(with_self=with_self)
         return Node.objects.filter(key__regex=pattern)
 
-    def get_all_children(self, with_self=False):
+    def get_all_children_pattern(self, with_self=False):
         pattern = r'^{0}:'.format(self.key)
         if with_self:
             pattern += r'|^{0}$'.format(self.key)
+        return pattern
+
+    def get_all_children(self, with_self=False):
+        pattern = self.get_all_children_pattern(with_self=with_self)
         children = Node.objects.filter(key__regex=pattern)
         return children
 
@@ -290,14 +298,16 @@ class NodeAssetsMixin:
         return self.get_all_assets().valid()
 
     @classmethod
-    def get_nodes_all_assets(cls, nodes_keys):
+    def get_nodes_all_assets(cls, nodes_keys, extra_assets_ids=None):
         from .asset import Asset
         nodes_keys = cls.clean_children_keys(nodes_keys)
-        pattern = set()
+        assets_ids = set()
         for key in nodes_keys:
-            pattern.add(r'^{0}$|^{0}:'.format(key))
-        pattern = '|'.join(list(pattern))
-        return Asset.objects.filter(nodes__key__regex=pattern)
+            node_assets_ids = cls.tree().all_assets(key)
+            assets_ids.update(set(node_assets_ids))
+        if extra_assets_ids:
+            assets_ids.update(set(extra_assets_ids))
+        return Asset.objects.filter(id__in=assets_ids)
 
 
 class SomeNodesMixin:

apps/assets/serializers/__init__.py

@@ -9,3 +9,4 @@ from .node import *
 from .domain import *
 from .cmd_filter import *
 from .asset_user import *
+from .gathered_user import *

apps/assets/serializers/asset_user.py

@@ -53,6 +53,7 @@ class AssetUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         if not validated_data.get("name") and validated_data.get("username"):
             validated_data["name"] = validated_data["username"]
         instance = AssetUserManager.create(**validated_data)
+        instance.set_version_and_latest()
         return instance
 
 

apps/assets/serializers/gathered_user.py

+# -*- coding: utf-8 -*-
+#
+
+from django.utils.translation import ugettext_lazy as _
+
+from orgs.mixins.serializers import OrgResourceModelSerializerMixin
+from ..models import GatheredUser
+
+
+class GatheredUserSerializer(OrgResourceModelSerializerMixin):
+    class Meta:
+        model = GatheredUser
+        fields = [
+            'id', 'asset', 'hostname', 'ip', 'username',
+            'present', 'date_created', 'date_updated'
+        ]
+        read_only_fields = fields
+        labels = {
+            'hostname': _("Hostname"),
+            'ip': "IP"
+        }

apps/assets/signals_handler.py

@@ -7,13 +7,14 @@ from django.db.models.signals import (
 from django.db.models.aggregates import Count
 from django.dispatch import receiver
 
-from common.utils import get_logger
+from common.utils import get_logger, timeit
 from common.decorator import on_transaction_commit
-from .models import Asset, SystemUser, Node, AuthBook
+from .models import Asset, SystemUser, Node
 from .tasks import (
     update_assets_hardware_info_util,
     test_asset_connectivity_util,
-    push_system_user_to_assets
+    push_system_user_to_assets,
+    add_nodes_assets_to_system_users
 )
 
 
@@ -99,7 +100,7 @@ def on_system_user_nodes_change(sender, instance=None, action=None, model=None,
     """
     if action != "post_add":
         return
-    logger.info("System user `{}` nodes update signal recv".format(instance))
+    logger.info("System user nodes update signal recv: {}".format(instance))
 
     queryset = model.objects.filter(pk__in=pk_set)
     if model == Node:
@@ -108,9 +109,7 @@ def on_system_user_nodes_change(sender, instance=None, action=None, model=None,
     else:
         nodes_keys = [instance.key]
         system_users = queryset
-    assets = Node.get_nodes_all_assets(nodes_keys).values_list('id', flat=True)
-    for system_user in system_users:
-        system_user.assets.add(*tuple(assets))
+    add_nodes_assets_to_system_users.delay(nodes_keys, system_users)
 
 
 @receiver(m2m_changed, sender=Asset.nodes.through)
@@ -190,10 +189,3 @@ def on_asset_nodes_remove(sender, instance=None, action='', model=None,
 def on_node_update_or_created(sender, **kwargs):
     # 刷新节点
     Node.refresh_nodes()
-
-
-@receiver(post_save, sender=AuthBook)
-def on_auth_book_created(sender, instance=None, created=False, **kwargs):
-    if created:
-        logger.debug('Receive create auth book object signal.')
-        instance.set_version_and_latest()

apps/assets/tasks/__init__.py

+# -*- coding: utf-8 -*-
+#
+from .utils import *
+from .common import *
+from .admin_user_connectivity import *
+from .asset_connectivity import *
+from .asset_user_connectivity import *
+from .gather_asset_users import *
+from .gather_asset_hardware_info import *
+from .push_system_user import *
+from .system_user_connectivity import *

apps/assets/tasks/admin_user_connectivity.py

+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+from django.core.cache import cache
+
+from common.utils import get_logger
+from ops.celery.decorator import register_as_period_task
+
+from ..models import AdminUser
+from .utils import clean_hosts
+from .asset_connectivity import test_asset_connectivity_util
+from . import const
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'test_admin_user_connectivity_util', 'test_admin_user_connectivity_manual',
+    'test_admin_user_connectivity_period'
+]
+
+
+@shared_task(queue="ansible")
+def test_admin_user_connectivity_util(admin_user, task_name):
+    """
+    Test asset admin user can connect or not. Using ansible api do that
+    :param admin_user:
+    :param task_name:
+    :return:
+    """
+    assets = admin_user.get_related_assets()
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+    summary = test_asset_connectivity_util(hosts, task_name)
+    return summary
+
+
+@shared_task(queue="ansible")
+@register_as_period_task(interval=3600)
+def test_admin_user_connectivity_period():
+    """
+    A period task that update the ansible task period
+    """
+    if const.PERIOD_TASK_ENABLED:
+        logger.debug('Period task off, skip')
+        return
+    key = '_JMS_TEST_ADMIN_USER_CONNECTIVITY_PERIOD'
+    prev_execute_time = cache.get(key)
+    if prev_execute_time:
+        logger.debug("Test admin user connectivity, less than 40 minutes, skip")
+        return
+    cache.set(key, 1, 60*40)
+    admin_users = AdminUser.objects.all()
+    for admin_user in admin_users:
+        task_name = _("Test admin user connectivity period: {}").format(admin_user.name)
+        test_admin_user_connectivity_util(admin_user, task_name)
+    cache.set(key, 1, 60*40)
+
+
+@shared_task(queue="ansible")
+def test_admin_user_connectivity_manual(admin_user):
+    task_name = _("Test admin user connectivity: {}").format(admin_user.name)
+    test_admin_user_connectivity_util(admin_user, task_name)
+    return True

apps/assets/tasks/asset_connectivity.py

+# ~*~ coding: utf-8 ~*~
+from collections import defaultdict
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+from ..models.utils import Connectivity
+from . import const
+from .utils import clean_hosts
+
+
+logger = get_logger(__file__)
+__all__ = ['test_asset_connectivity_util', 'test_asset_connectivity_manual']
+
+
+@shared_task(queue="ansible")
+def test_asset_connectivity_util(assets, task_name=None):
+    from ops.utils import update_or_create_ansible_task
+
+    if task_name is None:
+        task_name = _("Test assets connectivity")
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.TEST_ADMIN_USER_CONN_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.TEST_WINDOWS_ADMIN_USER_CONN_TASKS
+        }
+    }
+    for host in hosts:
+        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(host)
+
+    results_summary = dict(
+        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
+    )
+    created_by = assets[0].org_id
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
+            created_by=created_by,
+        )
+        raw, summary = task.run()
+        success = summary.get('success', False)
+        contacted = summary.get('contacted', {})
+        dark = summary.get('dark', {})
+
+        results_summary['success'] &= success
+        results_summary['contacted'].update(contacted)
+        results_summary['dark'].update(dark)
+
+    for asset in assets:
+        if asset.hostname in results_summary.get('dark', {}).keys():
+            asset.connectivity = Connectivity.unreachable()
+        elif asset.hostname in results_summary.get('contacted', {}).keys():
+            asset.connectivity = Connectivity.reachable()
+        else:
+            asset.connectivity = Connectivity.unknown()
+    return results_summary
+
+
+@shared_task(queue="ansible")
+def test_asset_connectivity_manual(asset):
+    task_name = _("Test assets connectivity: {}").format(asset)
+    summary = test_asset_connectivity_util([asset], task_name=task_name)
+
+    if summary.get('dark'):
+        return False, summary['dark']
+    else:
+        return True, ""

apps/assets/tasks/asset_user_connectivity.py

+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+from . import const
+from .utils import check_asset_can_run_ansible
+
+
+logger = get_logger(__file__)
+
+
+__all__ = [
+    'test_asset_user_connectivity_util', 'test_asset_users_connectivity_manual',
+    'get_test_asset_user_connectivity_tasks',
+]
+
+
+def get_test_asset_user_connectivity_tasks(asset):
+    if asset.is_unixlike():
+        tasks = const.TEST_ASSET_USER_CONN_TASKS
+    elif asset.is_windows():
+        tasks = const.TEST_WINDOWS_ASSET_USER_CONN_TASKS
+    else:
+        msg = _(
+            "The asset {} system platform {} does not "
+            "support run Ansible tasks".format(asset.hostname, asset.platform)
+        )
+        logger.info(msg)
+        tasks = []
+    return tasks
+
+
+@shared_task(queue="ansible")
+def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False):
+    """
+    :param asset_user: <AuthBook>对象
+    :param task_name:
+    :param run_as_admin:
+    :return:
+    """
+    from ops.utils import update_or_create_ansible_task
+
+    if not check_asset_can_run_ansible(asset_user.asset):
+        return
+
+    tasks = get_test_asset_user_connectivity_tasks(asset_user.asset)
+    if not tasks:
+        logger.debug("No tasks ")
+        return
+
+    args = (task_name,)
+    kwargs = {
+        'hosts': [asset_user.asset], 'tasks': tasks,
+        'pattern': 'all', 'options': const.TASK_OPTIONS,
+        'created_by': asset_user.org_id,
+    }
+    if run_as_admin:
+        kwargs["run_as_admin"] = True
+    else:
+        kwargs["run_as"] = asset_user.username
+    task, created = update_or_create_ansible_task(*args, **kwargs)
+    raw, summary = task.run()
+    asset_user.set_connectivity(summary)
+
+
+@shared_task(queue="ansible")
+def test_asset_users_connectivity_manual(asset_users, run_as_admin=False):
+    """
+    :param asset_users: <AuthBook>对象
+    """
+    for asset_user in asset_users:
+        task_name = _("Test asset user connectivity: {}").format(asset_user)
+        test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=run_as_admin)
+
+

apps/assets/tasks/common.py

+# -*- coding: utf-8 -*-
+#
+
+from celery import shared_task
+
+__all__ = ['add_nodes_assets_to_system_users']
+
+
+@shared_task
+def add_nodes_assets_to_system_users(nodes_keys, system_users):
+    from ..models import Node
+    assets = Node.get_nodes_all_assets(nodes_keys).values_list('id', flat=True)
+    for system_user in system_users:
+        system_user.assets.add(*tuple(assets))

apps/assets/const.py → apps/assets/tasks/const.py

 # -*- coding: utf-8 -*-
 #
+import os
 from django.utils.translation import ugettext_lazy as _
 
+
+PERIOD_TASK_ENABLED = os.environ.get("PERIOD_TASK", "on") == 'on'
+
 UPDATE_ASSETS_HARDWARE_TASKS = [
    {
        'name': "setup",
@@ -79,3 +83,22 @@ CONNECTIVITY_CHOICES = (
     (CONN_UNKNOWN, _("Unknown")),
 )
 
+GATHER_ASSET_USERS_TASKS = [
+    {
+        "name": "gather host users",
+        "action": {
+            "module": 'getent',
+            "args": "database=passwd"
+        },
+    },
+]
+
+GATHER_ASSET_USERS_TASKS_WINDOWS = [
+    {
+        "name": "gather windows host users",
+        "action": {
+            "module": 'win_shell',
+            "args": "net user"
+        }
+    }
+]

apps/assets/tasks/gather_asset_hardware_info.py

+# -*- coding: utf-8 -*-
+#
+import json
+import re
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import (
+    capacity_convert, sum_capacity, get_logger
+)
+from . import const
+from .utils import clean_hosts
+
+
+logger = get_logger(__file__)
+disk_pattern = re.compile(r'^hd|sd|xvd|vd|nv')
+__all__ = [
+    'update_assets_hardware_info_util', 'update_asset_hardware_info_manual',
+    'update_assets_hardware_info_period',
+]
+
+
+def set_assets_hardware_info(assets, result, **kwargs):
+    """
+    Using ops task run result, to update asset info
+
+    @shared_task must be exit, because we using it as a task callback, is must
+    be a celery task also
+    :param assets:
+    :param result:
+    :param kwargs: {task_name: ""}
+    :return:
+    """
+    result_raw = result[0]
+    assets_updated = []
+    success_result = result_raw.get('ok', {})
+
+    for asset in assets:
+        hostname = asset.hostname
+        info = success_result.get(hostname, {})
+        info = info.get('setup', {}).get('ansible_facts', {})
+        if not info:
+            logger.error(_("Get asset info failed: {}").format(hostname))
+            continue
+        ___vendor = info.get('ansible_system_vendor', 'Unknown')
+        ___model = info.get('ansible_product_name', 'Unknown')
+        ___sn = info.get('ansible_product_serial', 'Unknown')
+
+        for ___cpu_model in info.get('ansible_processor', []):
+            if ___cpu_model.endswith('GHz') or ___cpu_model.startswith("Intel"):
+                break
+        else:
+            ___cpu_model = 'Unknown'
+        ___cpu_model = ___cpu_model[:48]
+        ___cpu_count = info.get('ansible_processor_count', 0)
+        ___cpu_cores = info.get('ansible_processor_cores', None) or \
+                       len(info.get('ansible_processor', []))
+        ___cpu_vcpus = info.get('ansible_processor_vcpus', 0)
+        ___memory = '%s %s' % capacity_convert(
+            '{} MB'.format(info.get('ansible_memtotal_mb'))
+        )
+        disk_info = {}
+        for dev, dev_info in info.get('ansible_devices', {}).items():
+            if disk_pattern.match(dev) and dev_info['removable'] == '0':
+                disk_info[dev] = dev_info['size']
+        ___disk_total = '%.1f %s' % sum_capacity(disk_info.values())
+        ___disk_info = json.dumps(disk_info)
+
+        # ___platform = info.get('ansible_system', 'Unknown')
+        ___os = info.get('ansible_distribution', 'Unknown')
+        ___os_version = info.get('ansible_distribution_version', 'Unknown')
+        ___os_arch = info.get('ansible_architecture', 'Unknown')
+        ___hostname_raw = info.get('ansible_hostname', 'Unknown')
+
+        for k, v in locals().items():
+            if k.startswith('___'):
+                setattr(asset, k.strip('_'), v)
+        asset.save()
+        assets_updated.append(asset)
+    return assets_updated
+
+
+@shared_task
+def update_assets_hardware_info_util(assets, task_name=None):
+    """
+    Using ansible api to update asset hardware info
+    :param assets:  asset seq
+    :param task_name: task_name running
+    :return: result summary ['contacted': {}, 'dark': {}]
+    """
+    from ops.utils import update_or_create_ansible_task
+    if task_name is None:
+        task_name = _("Update some assets hardware info")
+    tasks = const.UPDATE_ASSETS_HARDWARE_TASKS
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+    created_by = str(assets[0].org_id)
+    task, created = update_or_create_ansible_task(
+        task_name, hosts=hosts, tasks=tasks, created_by=created_by,
+        pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
+    )
+    result = task.run()
+    set_assets_hardware_info(assets, result)
+    return True
+
+
+@shared_task(queue="ansible")
+def update_asset_hardware_info_manual(asset):
+    task_name = _("Update asset hardware info: {}").format(asset.hostname)
+    update_assets_hardware_info_util(
+        [asset], task_name=task_name
+    )
+
+
+@shared_task(queue="ansible")
+def update_assets_hardware_info_period():
+    """
+    Update asset hardware period task
+    :return:
+    """
+    if not const.PERIOD_TASK_ENABLED:
+        logger.debug("Period task disabled, update assets hardware info pass")
+        return

apps/assets/tasks/gather_asset_users.py

+# ~*~ coding: utf-8 ~*~
+
+import re
+from collections import defaultdict
+from celery import shared_task
+
+from django.utils.translation import ugettext as _
+
+from orgs.utils import tmp_to_org
+from common.utils import get_logger
+from ..models import GatheredUser, Node
+from .utils import clean_hosts
+from . import const
+
+__all__ = ['gather_asset_users', 'gather_nodes_asset_users']
+logger = get_logger(__name__)
+space = re.compile('\s+')
+ignore_login_shell = re.compile(r'nologin$|sync$|shutdown$|halt$')
+
+
+def parse_linux_result_to_users(result):
+    task_result = {}
+    for task_name, raw in result.items():
+        res = raw.get('ansible_facts', {}).get('getent_passwd')
+        if res:
+            task_result = res
+            break
+    if not task_result or not isinstance(task_result, dict):
+        return []
+    users = []
+    for username, attr in task_result.items():
+        if ignore_login_shell.search(attr[-1]):
+            continue
+        users.append(username)
+    return users
+
+
+def parse_windows_result_to_users(result):
+    task_result = []
+    for task_name, raw in result.items():
+        res = raw.get('stdout_lines', {})
+        if res:
+            task_result = res
+            break
+    if not task_result:
+        return []
+
+    users = []
+
+    for i in range(4):
+        task_result.pop(0)
+    for i in range(2):
+        task_result.pop()
+
+    for line in task_result:
+        user = space.split(line)
+        if user[0]:
+            users.append(user[0])
+    return users
+
+
+def add_asset_users(assets, results):
+    assets_map = {a.hostname: a for a in assets}
+    parser_map = {
+        'linux': parse_linux_result_to_users,
+        'windows': parse_windows_result_to_users
+    }
+
+    assets_users_map = {}
+
+    for platform, platform_results in results.items():
+        for hostname, res in platform_results.items():
+            parse = parser_map.get(platform)
+            users = parse(res)
+            logger.debug('Gathered host users: {} {}'.format(hostname, users))
+            asset = assets_map.get(hostname)
+            if not asset:
+                continue
+            assets_users_map[asset] = users
+
+    for asset, users in assets_users_map.items():
+        with tmp_to_org(asset.org_id):
+            GatheredUser.objects.filter(asset=asset, present=True)\
+                .update(present=False)
+            for username in users:
+                defaults = {'asset': asset, 'username': username, 'present': True}
+                GatheredUser.objects.update_or_create(
+                    defaults=defaults, asset=asset, username=username,
+                )
+
+
+@shared_task(queue="ansible")
+def gather_asset_users(assets, task_name=None):
+    from ops.utils import update_or_create_ansible_task
+    if task_name is None:
+        task_name = _("Gather assets users")
+    assets = clean_hosts(assets)
+    if not assets:
+        return
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.GATHER_ASSET_USERS_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.GATHER_ASSET_USERS_TASKS_WINDOWS
+        }
+    }
+    for asset in assets:
+        hosts_list = hosts_category['windows']['hosts'] if asset.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(asset)
+
+    results = {'linux': defaultdict(dict), 'windows': defaultdict(dict)}
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        _task_name = '{}: {}'.format(task_name, k)
+        task, created = update_or_create_ansible_task(
+            task_name=_task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS,
+            run_as_admin=True, created_by=value['hosts'][0].org_id,
+        )
+        raw, summary = task.run()
+        results[k].update(raw['ok'])
+    add_asset_users(assets, results)
+
+
+@shared_task(queue="ansible")
+def gather_nodes_asset_users(nodes_key):
+    assets = Node.get_nodes_all_assets(nodes_key)
+    assets_groups_by_100 = [assets[i:i+100] for i in range(0, len(assets), 100)]
+    for _assets in assets_groups_by_100:
+        gather_asset_users(_assets)

apps/assets/tasks/push_system_user.py

+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import encrypt_password, get_logger
+from . import const
+from .utils import clean_hosts_by_protocol, clean_hosts
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'push_system_user_util', 'push_system_user_to_assets',
+    'push_system_user_to_assets_manual', 'push_system_user_a_asset_manual',
+]
+
+
+def get_push_linux_system_user_tasks(system_user):
+    tasks = [
+        {
+            'name': 'Add user {}'.format(system_user.username),
+            'action': {
+                'module': 'user',
+                'args': 'name={} shell={} state=present'.format(
+                    system_user.username, system_user.shell,
+                ),
+            }
+        },
+        {
+            'name': 'Add group {}'.format(system_user.username),
+            'action': {
+                'module': 'group',
+                'args': 'name={} state=present'.format(
+                    system_user.username,
+                ),
+            }
+        },
+        {
+            'name': 'Check home dir exists',
+            'action': {
+                'module': 'stat',
+                'args': 'path=/home/{}'.format(system_user.username)
+            },
+            'register': 'home_existed'
+        },
+        {
+            'name': "Set home dir permission",
+            'action': {
+                'module': 'file',
+                'args': "path=/home/{0} owner={0} group={0} mode=700".format(system_user.username)
+            },
+            'when': 'home_existed.stat.exists == true'
+        }
+    ]
+    if system_user.password:
+        tasks.append({
+            'name': 'Set {} password'.format(system_user.username),
+            'action': {
+                'module': 'user',
+                'args': 'name={} shell={} state=present password={}'.format(
+                    system_user.username, system_user.shell,
+                    encrypt_password(system_user.password, salt="K3mIlKK"),
+                ),
+            }
+        })
+    if system_user.public_key:
+        tasks.append({
+            'name': 'Set {} authorized key'.format(system_user.username),
+            'action': {
+                'module': 'authorized_key',
+                'args': "user={} state=present key='{}'".format(
+                    system_user.username, system_user.public_key
+                )
+            }
+        })
+    if system_user.sudo:
+        sudo = system_user.sudo.replace('\r\n', '\n').replace('\r', '\n')
+        sudo_list = sudo.split('\n')
+        sudo_tmp = []
+        for s in sudo_list:
+            sudo_tmp.append(s.strip(','))
+        sudo = ','.join(sudo_tmp)
+        tasks.append({
+            'name': 'Set {} sudo setting'.format(system_user.username),
+            'action': {
+                'module': 'lineinfile',
+                'args': "dest=/etc/sudoers state=present regexp='^{0} ALL=' "
+                        "line='{0} ALL=(ALL) NOPASSWD: {1}' "
+                        "validate='visudo -cf %s'".format(
+                    system_user.username, sudo,
+                )
+            }
+        })
+
+    return tasks
+
+
+def get_push_windows_system_user_tasks(system_user):
+    tasks = []
+    if not system_user.password:
+        return tasks
+    tasks.append({
+        'name': 'Add user {}'.format(system_user.username),
+        'action': {
+            'module': 'win_user',
+            'args': 'fullname={} '
+                    'name={} '
+                    'password={} '
+                    'state=present '
+                    'update_password=always '
+                    'password_expired=no '
+                    'password_never_expires=yes '
+                    'groups="Users,Remote Desktop Users" '
+                    'groups_action=add '
+                    ''.format(system_user.name,
+                              system_user.username,
+                              system_user.password),
+        }
+    })
+    return tasks
+
+
+def get_push_system_user_tasks(host, system_user):
+    if host.is_unixlike():
+        tasks = get_push_linux_system_user_tasks(system_user)
+    elif host.is_windows():
+        tasks = get_push_windows_system_user_tasks(system_user)
+    else:
+        msg = _(
+            "The asset {} system platform {} does not "
+            "support run Ansible tasks".format(host.hostname, host.platform)
+        )
+        logger.info(msg)
+        tasks = []
+    return tasks
+
+
+@shared_task(queue="ansible")
+def push_system_user_util(system_user, assets, task_name):
+    from ops.utils import update_or_create_ansible_task
+    if not system_user.is_need_push():
+        msg = _("Push system user task skip, auto push not enable or "
+                "protocol is not ssh or rdp: {}").format(system_user.name)
+        logger.info(msg)
+        return {}
+
+    # Set root as system user is dangerous
+    if system_user.username.lower() in ["root", "administrator"]:
+        msg = _("For security, do not push user {}".format(system_user.username))
+        logger.info(msg)
+        return {}
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts = clean_hosts_by_protocol(system_user, hosts)
+    if not hosts:
+        return {}
+
+    for host in hosts:
+        system_user.load_specific_asset_auth(host)
+        tasks = get_push_system_user_tasks(host, system_user)
+        if not tasks:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=[host], tasks=tasks, pattern='all',
+            options=const.TASK_OPTIONS, run_as_admin=True,
+            created_by=system_user.org_id,
+        )
+        task.run()
+
+
+@shared_task(queue="ansible")
+def push_system_user_to_assets_manual(system_user):
+    assets = system_user.get_all_assets()
+    task_name = _("Push system users to assets: {}").format(system_user.name)
+    return push_system_user_util(system_user, assets, task_name=task_name)
+
+
+@shared_task(queue="ansible")
+def push_system_user_a_asset_manual(system_user, asset):
+    task_name = _("Push system users to asset: {} => {}").format(
+        system_user.name, asset
+    )
+    return push_system_user_util(system_user, [asset], task_name=task_name)
+
+
+@shared_task(queue="ansible")
+def push_system_user_to_assets(system_user, assets):
+    task_name = _("Push system users to assets: {}").format(system_user.name)
+    return push_system_user_util(system_user, assets, task_name)
+
+
+
+# @shared_task
+# @register_as_period_task(interval=3600)
+# @after_app_ready_start
+# @after_app_shutdown_clean_periodic
+# def push_system_user_period():
+#     for system_user in SystemUser.objects.all():
+#         push_system_user_related_nodes(system_user)
\ No newline at end of file

apps/assets/tasks/system_user_connectivity.py

+
+from collections import defaultdict
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+
+from ..models import SystemUser
+from . import const
+from .utils import clean_hosts, clean_hosts_by_protocol
+
+logger = get_logger(__name__)
+__all__ = [
+    'test_system_user_connectivity_util', 'test_system_user_connectivity_manual',
+    'test_system_user_connectivity_period', 'test_system_user_connectivity_a_asset',
+]
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_util(system_user, assets, task_name):
+    """
+    Test system cant connect his assets or not.
+    :param system_user:
+    :param assets:
+    :param task_name:
+    :return:
+    """
+    from ops.utils import update_or_create_ansible_task
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts = clean_hosts_by_protocol(system_user, hosts)
+    if not hosts:
+        return {}
+
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.TEST_SYSTEM_USER_CONN_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.TEST_WINDOWS_SYSTEM_USER_CONN_TASKS
+        }
+    }
+    for host in hosts:
+        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(host)
+
+    results_summary = dict(
+        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
+    )
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS,
+            run_as=system_user.username, created_by=system_user.org_id,
+        )
+        raw, summary = task.run()
+        success = summary.get('success', False)
+        contacted = summary.get('contacted', {})
+        dark = summary.get('dark', {})
+
+        results_summary['success'] &= success
+        results_summary['contacted'].update(contacted)
+        results_summary['dark'].update(dark)
+
+    system_user.set_connectivity(results_summary)
+    return results_summary
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_manual(system_user):
+    task_name = _("Test system user connectivity: {}").format(system_user)
+    assets = system_user.get_all_assets()
+    return test_system_user_connectivity_util(system_user, assets, task_name)
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_a_asset(system_user, asset):
+    task_name = _("Test system user connectivity: {} => {}").format(
+        system_user, asset
+    )
+    return test_system_user_connectivity_util(system_user, [asset], task_name)
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_period():
+    if not const.PERIOD_TASK_ENABLED:
+        logger.debug("Period task disabled, test system user connectivity pass")
+        return
+    system_users = SystemUser.objects.all()
+    for system_user in system_users:
+        task_name = _("Test system user connectivity period: {}").format(system_user)
+        assets = system_user.get_all_assets()
+        test_system_user_connectivity_util(system_user, assets, task_name)

apps/assets/tasks/utils.py

+# -*- coding: utf-8 -*-
+#
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'check_asset_can_run_ansible', 'clean_hosts', 'clean_hosts_by_protocol'
+]
+
+
+def check_asset_can_run_ansible(asset):
+    if not asset.is_active:
+        msg = _("Asset has been disabled, skipped: {}").format(asset)
+        logger.info(msg)
+        return False
+    if not asset.is_support_ansible():
+        msg = _("Asset may not be support ansible, skipped: {}").format(asset)
+        logger.info(msg)
+        return False
+    return True
+
+
+def clean_hosts(assets):
+    clean_assets = []
+    for asset in assets:
+        if not check_asset_can_run_ansible(asset):
+            continue
+        clean_assets.append(asset)
+    if not clean_assets:
+        logger.info(_("No assets matched, stop task"))
+    return clean_assets
+
+
+def clean_hosts_by_protocol(system_user, assets):
+    hosts = [
+        asset for asset in assets
+        if asset.has_protocol(system_user.protocol)
+    ]
+    if not hosts:
+        msg = _("No assets matched related system user protocol, stop task")
+        logger.info(msg)
+    return hosts

apps/assets/templates/assets/asset_list.html

@@ -85,7 +85,7 @@
                   <button data-toggle="dropdown" class="btn btn-default btn-sm dropdown-toggle">{% trans 'Label' %} <span class="caret"></span></button>
                   <ul class="dropdown-menu labels">
                       {% for label in labels %}
-                          <li><a style="font-weight: bolder">{{ label.name }}:{{ label.value }}</a></li>
+                          <li><a style="font-weight: bolder">{{ label.name }}#{{ label.value }}</a></li>
                       {% endfor %}
                   </ul>
                </div>
@@ -171,9 +171,13 @@ function initTable() {
         ],
         ajax_url: '{% url "api-assets:asset-list" %}',
         columns: [
-            {data: "id"}, {data: "hostname" }, {data: "ip" },
+            {data: "id"}, {data: "hostname"}, {data: "ip"},
             {data: "cpu_cores", orderable: false},
-            {data: "connectivity", orderable: false}, {data: "id", orderable: false }
+            {
+                data: "connectivity",
+                orderable: false,
+                width: '60px'
+            }, {data: "id", orderable: false}
         ],
         op_html: $('#actions').html()
     };
@@ -271,7 +275,7 @@ $(document).ready(function(){
     setAssetModalOptions(modalOption);
 })
 .on('click', '.labels li', function () {
-    var val = $(this).text();
+    var val = 'label:' + $(this).text();
     $("#asset_list_table_filter input").val(val);
     asset_table.search(val).draw();
  })

apps/assets/templates/assets/user_asset_list.html

@@ -24,7 +24,7 @@
 var treeUrl = "{% url 'api-perms:my-nodes-children-as-tree' %}?&cache_policy=1";
 var assetTableUrl = "{% url 'api-perms:my-assets' %}?cache_policy=1";
 var selectUrl = '{% url "api-perms:my-node-assets" node_id=DEFAULT_PK %}?cache_policy=1&all=1';
-var systemUsersUrl = "{% url 'api-perms:my-asset-system-users' asset_id=DEFAULT_PK %}";
+var systemUsersUrl = "{% url 'api-perms:my-asset-system-users' asset_id=DEFAULT_PK %}?cache_policy=1";
 var showAssetHref = false; // Need input default true
 var actions = {
     targets: 4, createdCell: function (td, cellData) {

apps/assets/urls/api_urls.py

@@ -21,6 +21,7 @@ router.register(r'gateways', api.GatewayViewSet, 'gateway')
 router.register(r'cmd-filters', api.CommandFilterViewSet, 'cmd-filter')
 router.register(r'asset-users', api.AssetUserViewSet, 'asset-user')
 router.register(r'asset-users-info', api.AssetUserExportViewSet, 'asset-user-info')
+router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user')
 
 cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filters', lookup='filter')
 cmd_filter_router.register(r'rules', api.CommandFilterRuleViewSet, 'cmd-filter-rule')

apps/assets/utils.py

@@ -24,37 +24,6 @@ def get_system_user_by_id(id):
     return system_user
 
 
-class LabelFilterMixin:
-    def get_filter_labels_ids(self):
-        query_params = self.request.query_params
-        query_keys = query_params.keys()
-        all_label_keys = Label.objects.values_list('name', flat=True)
-        valid_keys = set(all_label_keys) & set(query_keys)
-
-        if not valid_keys:
-            return []
-
-        labels_query = [
-            {"name": key, "value": query_params[key]}
-            for key in valid_keys
-        ]
-        args = [Q(**kwargs) for kwargs in labels_query]
-        args = reduce(lambda x, y: x | y, args)
-        labels_id = Label.objects.filter(args).values_list('id', flat=True)
-        return labels_id
-
-
-class LabelFilter(LabelFilterMixin):
-    def filter_queryset(self, queryset):
-        queryset = super().filter_queryset(queryset)
-        labels_ids = self.get_filter_labels_ids()
-        if not labels_ids:
-            return queryset
-        for labels_id in labels_ids:
-            queryset = queryset.filter(labels=labels_id)
-        return queryset
-
-
 class TreeService(Tree):
     tag_sep = ' / '
     cache_key = '_NODE_FULL_TREE'

apps/common/filters.py

 # -*- coding: utf-8 -*-
 #
+import coreapi
 from rest_framework import filters
 from rest_framework.fields import DateTimeField
 from rest_framework.serializers import ValidationError
+from django.core.cache import cache
 import logging
 
-__all__ = ["DatetimeRangeFilter"]
+from . import const
+
+__all__ = ["DatetimeRangeFilter", "IDSpmFilter", "CustomFilter"]
 
 
 class DatetimeRangeFilter(filters.BaseFilterBackend):
@@ -40,3 +44,52 @@ class DatetimeRangeFilter(filters.BaseFilterBackend):
         if kwargs:
             queryset = queryset.filter(**kwargs)
         return queryset
+
+
+class IDSpmFilter(filters.BaseFilterBackend):
+    def get_schema_fields(self, view):
+        return [
+            coreapi.Field(
+                name='spm', location='query', required=False,
+                type='string', example='',
+                description='Pre post objects id get spm id, then using filter'
+            )
+        ]
+
+    def filter_queryset(self, request, queryset, view):
+        spm = request.query_params.get('spm')
+        if not spm:
+            return queryset
+        cache_key = const.KEY_CACHE_RESOURCES_ID.format(spm)
+        resources_id = cache.get(cache_key)
+        if not resources_id or not isinstance(resources_id, list):
+            queryset = queryset.none()
+            return queryset
+        queryset = queryset.filter(id__in=resources_id)
+        return queryset
+
+
+class CustomFilter(filters.BaseFilterBackend):
+
+    def get_schema_fields(self, view):
+        fields = []
+        defaults = dict(
+            location='query', required=False,
+            type='string', example='',
+            description=''
+        )
+        if not hasattr(view, 'custom_filter_fields'):
+            return []
+
+        for field in view.custom_filter_fields:
+            if isinstance(field, str):
+                defaults['name'] = field
+            elif isinstance(field, dict):
+                defaults.update(field)
+            else:
+                continue
+            fields.append(coreapi.Field(**defaults))
+        return fields
+
+    def filter_queryset(self, request, queryset, view):
+        return queryset

apps/common/mixins/api.py

 # -*- coding: utf-8 -*-
 #
 from django.http import JsonResponse
-from django.core.cache import cache
-from django.utils.translation import ugettext_lazy as _
-from django.contrib import messages
+from rest_framework.settings import api_settings
 
-from ..const import KEY_CACHE_RESOURCES_ID
+from ..filters import IDSpmFilter, CustomFilter
 
 __all__ = [
-    "JSONResponseMixin", "IDInCacheFilterMixin", "IDExportFilterMixin",
-    "IDInFilterMixin", "ApiMessageMixin"
+    "JSONResponseMixin", "CommonApiMixin",
+    "IDSpmFilterMixin", "CommonApiMixin",
 ]
 
 
@@ -20,69 +18,31 @@ class JSONResponseMixin(object):
         return JsonResponse(context)
 
 
-class IDInFilterMixin(object):
-    def filter_queryset(self, queryset):
-        queryset = super(IDInFilterMixin, self).filter_queryset(queryset)
-        id_list = self.request.query_params.get('id__in')
-        if id_list:
-            import json
-            try:
-                ids = json.loads(id_list)
-            except Exception as e:
-                return queryset
-            if isinstance(ids, list):
-                queryset = queryset.filter(id__in=ids)
-        return queryset
+class IDSpmFilterMixin:
+    def get_filter_backends(self):
+        backends = super().get_filter_backends()
+        backends.append(IDSpmFilter)
+        return backends
 
 
-class IDInCacheFilterMixin(object):
-
-    def filter_queryset(self, queryset):
-        queryset = super().filter_queryset(queryset)
-        spm = self.request.query_params.get('spm')
-        if not spm:
-            return queryset
-        cache_key = KEY_CACHE_RESOURCES_ID.format(spm)
-        resources_id = cache.get(cache_key)
-        if not resources_id or not isinstance(resources_id, list):
-            queryset = queryset.none()
-            return queryset
-        queryset = queryset.filter(id__in=resources_id)
-        return queryset
+class ExtraFilterFieldsMixin:
+    default_added_filters = [CustomFilter, IDSpmFilter]
+    filter_backends = api_settings.DEFAULT_FILTER_BACKENDS
+    extra_filter_fields = []
+    extra_filter_backends = []
 
+    def get_filter_backends(self):
+        if self.filter_backends != self.__class__.filter_backends:
+            return self.filter_backends
+        return list(self.filter_backends) + \
+               self.default_added_filters + \
+               list(self.extra_filter_backends)
 
-class IDExportFilterMixin(object):
     def filter_queryset(self, queryset):
-        # 下载导入模版
-        if self.request.query_params.get('template') == 'import':
-            return []
-        else:
-            return super(IDExportFilterMixin, self).filter_queryset(queryset)
-
-
-class ApiMessageMixin:
-    success_message = _("%(name)s was %(action)s successfully")
-    _action_map = {"create": _("create"), "update": _("update")}
+        for backend in self.get_filter_backends():
+            queryset = backend().filter_queryset(self.request, queryset, self)
+        return queryset
 
-    def get_success_message(self, cleaned_data):
-        if not isinstance(cleaned_data, dict):
-            return ''
-        data = {k: v for k, v in cleaned_data.items()}
-        action = getattr(self, "action", "create")
-        data["action"] = self._action_map.get(action)
-        try:
-            message = self.success_message % data
-        except:
-            message = ''
-        return message
 
-    def dispatch(self, request, *args, **kwargs):
-        resp = super().dispatch(request, *args, **kwargs)
-        if request.method.lower() in ("get", "delete", "patch"):
-            return resp
-        if resp.status_code >= 400:
-            return resp
-        message = self.get_success_message(resp.data)
-        if message:
-            messages.success(request, message)
-        return resp
+class CommonApiMixin(ExtraFilterFieldsMixin):
+    pass

apps/common/utils/common.py

@@ -8,7 +8,6 @@ import datetime
 import uuid
 from functools import wraps
 import time
-import copy
 import ipaddress
 
 
@@ -199,3 +198,31 @@ def timeit(func):
         logger.debug(msg)
         return result
     return wrapper
+
+
+def group_obj_by_count(objs, count=50):
+    objs_grouped = [
+        objs[i:i + count] for i in range(0, len(objs), count)
+    ]
+    return objs_grouped
+
+
+def dict_get_any(d, keys):
+    for key in keys:
+        value = d.get(key)
+        if value:
+            return value
+    return None
+
+
+class lazyproperty:
+    def __init__(self, func):
+        self.func = func
+
+    def __get__(self, instance, cls):
+        if instance is None:
+            return self
+        else:
+            value = self.func(instance)
+            setattr(instance, self.func.__name__, value)
+            return value
\ No newline at end of file

apps/jumpserver/celery_flower.py

+# -*- coding: utf-8 -*-
+#
+from django.http import HttpResponse
+from django.conf import settings
+from django.utils.translation import ugettext as _
+from django.views.decorators.csrf import csrf_exempt
+
+from proxy.views import proxy_view
+
+flower_url = settings.FLOWER_URL
+
+
+@csrf_exempt
+def celery_flower_view(request, path):
+    if not request.user.is_superuser:
+        return HttpResponse("Forbidden")
+    remote_url = 'http://{}/{}'.format(flower_url, path)
+    try:
+        response = proxy_view(request, remote_url)
+    except Exception as e:
+        msg = _("<h1>Flow service unavailable, check it</h1>") + \
+              '<br><br> <div>{}</div>'.format(e)
+        response = HttpResponse(msg)
+    return response
+

apps/jumpserver/conf.py

@@ -382,6 +382,8 @@ defaults = {
     'SYSLOG_ADDR': '',  # '192.168.0.1:514'
     'SYSLOG_FACILITY': 'user',
     'PERM_SINGLE_ASSET_TO_UNGROUP_NODE': False,
+    'WINDOWS_SSH_DEFAULT_SHELL': 'cmd',
+    'FLOWER_URL': "127.0.0.1:5555"
 }
 
 

apps/jumpserver/settings.py

@@ -622,3 +622,5 @@ ASSETS_PERM_CACHE_TIME = CONFIG.ASSETS_PERM_CACHE_TIME
 BACKEND_ASSET_USER_AUTH_VAULT = False
 
 PERM_SINGLE_ASSET_TO_UNGROUP_NODE = CONFIG.PERM_SINGLE_ASSET_TO_UNGROUP_NODE
+WINDOWS_SSH_DEFAULT_SHELL = CONFIG.WINDOWS_SSH_DEFAULT_SHELL
+FLOWER_URL = CONFIG.FLOWER_URL

apps/jumpserver/swagger.py

@@ -33,6 +33,21 @@ class CustomSwaggerAutoSchema(SwaggerAutoSchema):
         operation.summary = operation.operation_id
         return operation
 
+    def get_filter_parameters(self):
+        if not self.should_filter():
+            return []
+
+        fields = []
+        if hasattr(self.view, 'get_filter_backends'):
+            backends = self.view.get_filter_backends()
+        elif hasattr(self.view, 'filter_backends'):
+            backends = self.view.filter_backends
+        else:
+            backends = []
+        for filter_backend in backends:
+            fields += self.probe_inspectors(self.filter_inspectors, 'get_filter_parameters', filter_backend()) or []
+        return fields
+
 
 def get_swagger_view(version='v1'):
     from .urls import api_v1, api_v2

apps/jumpserver/urls.py

@@ -7,7 +7,9 @@ from django.conf.urls.static import static
 from django.conf.urls.i18n import i18n_patterns
 from django.views.i18n import JavaScriptCatalog
 
-from .views import IndexView, LunaView, I18NView, HealthCheckView, redirect_format_api
+# from .views import IndexView, LunaView, I18NView, HealthCheckView, redirect_format_api
+from . import views
+from .celery_flower import celery_flower_view
 from .swagger import get_swagger_view
 
 api_v1 = [
@@ -40,6 +42,7 @@ app_view_patterns = [
     path('orgs/', include('orgs.urls.views_urls', namespace='orgs')),
     path('auth/', include('authentication.urls.view_urls'), name='auth'),
     path('applications/', include('applications.urls.views_urls', namespace='applications')),
+    re_path(r'flower/(?P<path>.*)', celery_flower_view, name='flower-view'),
 ]
 
 
@@ -57,13 +60,13 @@ js_i18n_patterns = i18n_patterns(
 
 
 urlpatterns = [
-    path('', IndexView.as_view(), name='index'),
+    path('', views.IndexView.as_view(), name='index'),
     path('api/v1/', include(api_v1)),
     path('api/v2/', include(api_v2)),
-    re_path('api/(?P<app>\w+)/(?P<version>v\d)/.*', redirect_format_api),
-    path('api/health/', HealthCheckView.as_view(), name="health"),
-    path('luna/', LunaView.as_view(), name='luna-view'),
-    path('i18n/<str:lang>/', I18NView.as_view(), name='i18n-switch'),
+    re_path('api/(?P<app>\w+)/(?P<version>v\d)/.*', views.redirect_format_api),
+    path('api/health/', views.HealthCheckView.as_view(), name="health"),
+    path('luna/', views.LunaView.as_view(), name='luna-view'),
+    path('i18n/<str:lang>/', views.I18NView.as_view(), name='i18n-switch'),
     path('settings/', include('settings.urls.view_urls', namespace='settings')),
 
     # External apps url

apps/jumpserver/views.py

@@ -224,3 +224,6 @@ class HealthCheckView(APIView):
 
     def get(self, request):
         return JsonResponse({"status": 1, "time": int(time.time())})
+
+
+

apps/ops/celery/utils.py

@@ -6,7 +6,9 @@ import os
 from django.conf import settings
 from django.utils.timezone import get_current_timezone
 from django.db.utils import ProgrammingError, OperationalError
-from django_celery_beat.models import PeriodicTask, IntervalSchedule, CrontabSchedule
+from django_celery_beat.models import (
+    PeriodicTask, IntervalSchedule, CrontabSchedule, PeriodicTasks
+)
 
 
 def create_or_update_celery_periodic_tasks(tasks):
@@ -75,17 +77,20 @@ def create_or_update_celery_periodic_tasks(tasks):
         task = PeriodicTask.objects.update_or_create(
             defaults=defaults, name=name,
         )
+        PeriodicTasks.update_changed()
         return task
 
 
 def disable_celery_periodic_task(task_name):
     from django_celery_beat.models import PeriodicTask
     PeriodicTask.objects.filter(name=task_name).update(enabled=False)
+    PeriodicTasks.update_changed()
 
 
 def delete_celery_periodic_task(task_name):
     from django_celery_beat.models import PeriodicTask
     PeriodicTask.objects.filter(name=task_name).delete()
+    PeriodicTasks.update_changed()
 
 
 def get_celery_task_log_path(task_id):

apps/ops/inventory.py

 # -*- coding: utf-8 -*-
 #
 
+from django.conf import settings
 from .ansible.inventory import BaseInventory
 
 from common.utils import get_logger
@@ -14,6 +15,7 @@ logger = get_logger(__file__)
 
 
 class JMSBaseInventory(BaseInventory):
+    windows_ssh_default_ssh = settings.WINDOWS_SSH_DEFAULT_SHELL
 
     def convert_to_ansible(self, asset, run_as_admin=False):
         info = {
@@ -33,7 +35,7 @@ class JMSBaseInventory(BaseInventory):
         if asset.is_windows():
             info["vars"].update({
                 "ansible_connection": "ssh",
-                "ansible_shell_type": "cmd",
+                "ansible_shell_type": self.windows_ssh_default_ssh,
             })
         for label in asset.labels.all():
             info["vars"].update({
@@ -49,7 +51,7 @@ class JMSBaseInventory(BaseInventory):
     def make_proxy_command(asset):
         gateway = asset.domain.random_gateway()
         proxy_command_list = [
-            "ssh", "-p", str(gateway.port),
+            "ssh", "-o", "Port={}".format(gateway.port),
             "-o", "StrictHostKeyChecking=no",
             "{}@{}".format(gateway.username, gateway.ip),
             "-W", "%h:%p", "-q",

apps/ops/migrations/0008_auto_20190919_2100.py

+# Generated by Django 2.1.7 on 2019-09-19 13:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0007_auto_20190724_2002'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='date_updated',
+            field=models.DateTimeField(auto_now=True, verbose_name='Date updated'),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='date_created',
+            field=models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='Date created'),
+        ),
+        migrations.AlterModelOptions(
+            name='task',
+            options={'get_latest_by': 'date_created',
+                     'ordering': ('-date_updated',)},
+        ),
+    ]

apps/ops/serializers.py

@@ -19,7 +19,12 @@ class CeleryTaskSerializer(serializers.Serializer):
 class TaskSerializer(serializers.ModelSerializer):
     class Meta:
         model = Task
-        fields = '__all__'
+        fields = [
+            'id', 'name', 'interval', 'crontab', 'is_periodic',
+            'is_deleted', 'comment', 'created_by', 'date_created',
+            'versions', 'is_success', 'timedelta', 'assets_amount',
+            'date_updated', 'history_summary',
+        ]
 
 
 class AdHocSerializer(serializers.ModelSerializer):

apps/orgs/utils.py

@@ -18,6 +18,8 @@ def get_org_from_request(request):
 
 
 def set_current_org(org):
+    if isinstance(org, str):
+        org = Organization.get_instance(org)
     setattr(thread_local, 'current_org_id', org.id)
 
 

apps/terminal/serializers_v2/terminal.py

@@ -58,5 +58,5 @@ class TerminalSerializer(serializers.ModelSerializer):
 
 class TerminalRegistrationSerializer(serializers.Serializer):
     name = serializers.CharField(max_length=128)
-    comment = serializers.CharField(max_length=128)
+    comment = serializers.CharField(max_length=128, )
     service_account = ServiceAccountSerializer(read_only=True)