diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index f8ed14cb94297c54398621f48929937f6b9cf253..3e1c10ede71fb891eec4483ed837f113fa7beac0 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -20,6 +20,7 @@ from common.mixins import IDInCacheFilterMixin, ApiMessageMixin from common.utils import get_logger, get_object_or_none from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser +from orgs.mixins import OrgBulkModelViewSet from ..const import CACHE_KEY_ASSET_BULK_UPDATE_ID_PREFIX from ..models import Asset, AdminUser, Node from .. import serializers @@ -36,7 +37,7 @@ __all__ = [ ] -class AssetViewSet(IDInCacheFilterMixin, LabelFilter, ApiMessageMixin, BulkModelViewSet): +class AssetViewSet(LabelFilter, ApiMessageMixin, OrgBulkModelViewSet): """ API endpoint that allows Asset to be viewed or edited. """ @@ -100,11 +101,6 @@ class AssetViewSet(IDInCacheFilterMixin, LabelFilter, ApiMessageMixin, BulkModel queryset = self.filter_admin_user_id(queryset) return queryset - def get_queryset(self): - queryset = super().get_queryset().distinct() - queryset = self.get_serializer_class().setup_eager_loading(queryset) - return queryset - class AssetListUpdateApi(IDInCacheFilterMixin, ListBulkCreateUpdateDestroyAPIView): """ diff --git a/apps/assets/api/asset_user.py b/apps/assets/api/asset_user.py index 237517790da75ac2330e0fd59263afb2ca64376e..cf092dab06a1bf6e8fde2a0083f39089585cc5b9 100644 --- a/apps/assets/api/asset_user.py +++ b/apps/assets/api/asset_user.py @@ -127,15 +127,14 @@ class AssetUserAuthInfoApi(generics.RetrieveAPIView): return Response(serializer.data, status=status_code) def get_object(self): - username = self.request.GET.get('username') - asset_id = self.request.GET.get('asset_id') - prefer = self.request.GET.get("prefer") + query_params = self.request.query_params + username = query_params.get('username') + asset_id = query_params.get('asset_id') + prefer = query_params.get("prefer") asset = get_object_or_none(Asset, pk=asset_id) try: manger = AssetUserManager() - if prefer: - manger.prefer(prefer) - instance = manger.get(username, asset) + instance = manger.get(username, asset, prefer=prefer) except Exception as e: logger.error(e, exc_info=True) return None diff --git a/apps/assets/api/system_user.py b/apps/assets/api/system_user.py index f6398e974a02b9fb0c98384557c0cc4aeb11a032..8baacd4f82bfd5219eb89ac7616e4d5ad536d330 100644 --- a/apps/assets/api/system_user.py +++ b/apps/assets/api/system_user.py @@ -22,6 +22,7 @@ from rest_framework.pagination import LimitOffsetPagination from common.utils import get_logger from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser from common.mixins import IDInCacheFilterMixin +from orgs.mixins import OrgBulkModelViewSet from ..models import SystemUser, Asset from .. import serializers from ..tasks import push_system_user_to_assets_manual, \ @@ -39,7 +40,7 @@ __all__ = [ ] -class SystemUserViewSet(IDInCacheFilterMixin, BulkModelViewSet): +class SystemUserViewSet(OrgBulkModelViewSet): """ System user api set, for add,delete,update,list,retrieve resource """ diff --git a/apps/assets/backends/asset_user.py b/apps/assets/backends/asset_user.py index 62ab24b4d99c66f78e6475887fc7100468a89112..774172e6cfee6a40407ba698463c820351956224 100644 --- a/apps/assets/backends/asset_user.py +++ b/apps/assets/backends/asset_user.py @@ -14,6 +14,11 @@ class AssetUserBackend(BaseBackend): @classmethod def filter(cls, username=None, assets=None, **kwargs): queryset = cls.model.objects.all() + prefer_id = kwargs.get('prefer_id') + if prefer_id: + queryset = queryset.filter(id=prefer_id) + instances = cls.construct_authbook_objects(queryset, assets) + return instances if username: queryset = queryset.filter(username=username) if assets: diff --git a/apps/assets/backends/base.py b/apps/assets/backends/base.py index d6f30f94004d47b9b09f9b970d62fca3dfc1c37e..f45b0b3b8fb9015491c60d8f881f519d7a546cf6 100644 --- a/apps/assets/backends/base.py +++ b/apps/assets/backends/base.py @@ -7,11 +7,13 @@ from abc import abstractmethod class BaseBackend: @classmethod @abstractmethod - def filter(cls, username=None, assets=None, latest=True): + def filter(cls, username=None, assets=None, latest=True, prefer=None, prefer_id=None): """ :param username: 用户名 :param assets: 对象 :param latest: 是否是最新记录 + :param prefer: 优先使用 + :param prefer_id: 使用id :return: 元素为的可迭代对象( or ) """ pass diff --git a/apps/assets/backends/db.py b/apps/assets/backends/db.py index f37569e51e38a7f8880497df0898cd2a1ebba4ff..40fa414446eaa63acf1e31f2fc769d9ff4a24087 100644 --- a/apps/assets/backends/db.py +++ b/apps/assets/backends/db.py @@ -7,7 +7,7 @@ from .base import BaseBackend class AuthBookBackend(BaseBackend): @classmethod - def filter(cls, username=None, assets=None, latest=True): + def filter(cls, username=None, assets=None, latest=True, **kwargs): queryset = AuthBook.objects.all() if username is not None: queryset = queryset.filter(username=username) diff --git a/apps/assets/backends/manager.py b/apps/assets/backends/manager.py index 2cdd52b1e73c97d8d8ce814ea35110d555909390..6745ea44b58a63098159b4d2c98bedf7266d6754 100644 --- a/apps/assets/backends/manager.py +++ b/apps/assets/backends/manager.py @@ -30,24 +30,22 @@ class AssetUserManager: ) _prefer = "system_user" - _using = None - def filter(self, username=None, assets=None, latest=True): + def filter(self, username=None, assets=None, latest=True, prefer=None, prefer_id=None): if assets is not None and not assets: return AssetUserQuerySet([]) - if self._using: - backend = dict(self.backends).get(self._using) - if not backend: - return self.none() - instances = backend.filter(username=username, assets=assets, latest=latest) - return AssetUserQuerySet(instances) + if prefer: + self._prefer = prefer instances_map = {} instances = [] for name, backend in self.backends: + if name != "db" and self._prefer != name: + continue _instances = backend.filter( - username=username, assets=assets, latest=latest + username=username, assets=assets, latest=latest, + prefer=self._prefer, prefer_id=prefer_id, ) instances_map[name] = _instances @@ -64,12 +62,12 @@ class AssetUserManager: else: ordering.extend(["admin_user", "system_user"]) # 根据prefer决定优先使用系统用户或管理用户谁的 - ordering_instances = [instances_map.get(i) for i in ordering] + ordering_instances = [instances_map.get(i, []) for i in ordering] instances = self._merge_instances(*ordering_instances) return AssetUserQuerySet(instances) - def get(self, username, asset): - instances = self.filter(username, assets=[asset]) + def get(self, username, asset, **kwargs): + instances = self.filter(username, assets=[asset], **kwargs) if len(instances) == 1: return instances[0] elif len(instances) == 0: @@ -95,10 +93,6 @@ class AssetUserManager: self._prefer = s return self - def using(self, s): - self._using = s - return self - @staticmethod def none(): return AssetUserQuerySet() diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index aa6e639a6c961a20f1775143025ce3ad1a1666e2..8c5c156a667d3b567c608648c35142a66bd2672d 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -5,6 +5,7 @@ import uuid from hashlib import md5 import sshpubkeys +from django.core.cache import cache from django.db import models from django.utils.translation import ugettext_lazy as _ from django.conf import settings @@ -34,7 +35,10 @@ class AssetUser(OrgModelMixin): date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) - CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_ASSET_CONNECTIVITY_{}" + CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_{}_ASSET_CONNECTIVITY" + CONNECTIVITY_AMOUNT_CACHE_KEY = "ASSET_USER_{}_CONNECTIVITY_AMOUNT" + ASSETS_AMOUNT_CACHE_KEY = "ASSET_USER_{}_ASSETS_AMOUNT" + ASSET_USER_CACHE_TIME = 3600 * 24 _prefer = "system_user" @@ -67,6 +71,11 @@ class AssetUser(OrgModelMixin): pass return None + @property + def part_id(self): + i = '-'.join(str(self.id).split('-')[:3]) + return i + def get_related_assets(self): assets = self.assets.all() return assets @@ -97,10 +106,14 @@ class AssetUser(OrgModelMixin): self.set_asset_connectivity(asset, Connectivity.reachable()) else: self.set_asset_connectivity(asset, Connectivity.unknown()) + cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id) + cache.delete(cache_key) @property def connectivity(self): - assets = self.get_related_assets() + assets = self.get_related_assets()\ + .select_related('admin_user')\ + .only('id', 'hostname', 'admin_user') data = { 'unreachable': [], 'reachable': [], @@ -118,11 +131,25 @@ class AssetUser(OrgModelMixin): @property def connectivity_amount(self): - return {k: len(v) for k, v in self.connectivity.items()} + cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id) + amount = cache.get(cache_key) + if not amount: + connectivity = {k: len(v) for k, v in self.connectivity.items()} + cache.set(cache_key, connectivity, self.ASSET_USER_CACHE_TIME) + return amount @property def assets_amount(self): - return self.get_related_assets().count() + cache_key = self.ASSETS_AMOUNT_CACHE_KEY.format(self.id) + cached = cache.get(cache_key) + if not cached: + cached = self.get_related_assets().count() + cache.set(cache_key, cached, self.ASSET_USER_CACHE_TIME) + return cached + + def expire_assets_amount(self): + cache_key = self.ASSETS_AMOUNT_CACHE_KEY.format(self.id) + cache.delete(cache_key) def get_asset_connectivity(self, asset): i = self.generate_id_with_asset(asset) @@ -133,12 +160,15 @@ class AssetUser(OrgModelMixin): i = self.generate_id_with_asset(asset) key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i) Connectivity.set(key, c) + # 当为某个系统用户或管理用户设置的的时候,失效掉他们的连接数量 + amount_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id) + cache.delete(amount_key) def get_asset_user(self, asset): from ..backends import AssetUserManager try: manager = AssetUserManager().prefer(self._prefer) - other = manager.get(username=self.username, asset=asset) + other = manager.get(username=self.username, asset=asset, prefer_id=self.id) return other except Exception as e: logger.error(e, exc_info=True) @@ -150,9 +180,12 @@ class AssetUser(OrgModelMixin): self._merge_auth(instance) def _merge_auth(self, other): - self.password = other.password - self.public_key = other.public_key - self.private_key = other.private_key + if other.password: + self.password = other.password + if other.public_key: + self.public_key = other.public_key + if other.private_key: + self.private_key = other.private_key def clear_auth(self): self.password = '' @@ -185,7 +218,7 @@ class AssetUser(OrgModelMixin): } def generate_id_with_asset(self, asset): - user_id = str(self.id).split('-')[:3] + user_id = [self.part_id] asset_id = str(asset.id).split('-')[3:] ids = user_id + asset_id return '-'.join(ids) diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index ace1237cc45abcb8ad9172dcadbf939d8f8e5ffa..6190357dff17a5513c8e5f9a8f95008d8ca2b0a2 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -192,6 +192,7 @@ class AssetsAmountMixin: _assets_amount_cache_key = '_NODE_ASSETS_AMOUNT_{}' _assets_amount = None key = '' + cache_time = 3600 * 24 * 7 @property def assets_amount(self): @@ -213,7 +214,7 @@ class AssetsAmountMixin: def assets_amount(self, value): self._assets_amount = value cache_key = self._assets_amount_cache_key.format(self.key) - cache.set(cache_key, value) + cache.set(cache_key, value, self.cache_time) def expire_assets_amount(self): ancestor_keys = self.get_ancestor_keys(with_self=True) diff --git a/apps/assets/serializers/admin_user.py b/apps/assets/serializers/admin_user.py index d623148585f16e13198003a86cedb08db3d2b0c7..dbbb164065dd91b382a7a6e6cf49ec573a48f708 100644 --- a/apps/assets/serializers/admin_user.py +++ b/apps/assets/serializers/admin_user.py @@ -17,7 +17,7 @@ class AdminUserSerializer(BulkOrgResourceModelSerializer): """ class Meta: - # list_serializer_class = AdaptedBulkListSerializer + list_serializer_class = AdaptedBulkListSerializer model = AdminUser fields = [ 'id', 'name', 'username', 'password', 'private_key', 'public_key', diff --git a/apps/assets/serializers/asset.py b/apps/assets/serializers/asset.py index 6af18b84c07b6ef195f97f9e6b60387062702c9b..6cda758b81033a97d74960afa1ef9f5ef26b0555 100644 --- a/apps/assets/serializers/asset.py +++ b/apps/assets/serializers/asset.py @@ -76,8 +76,8 @@ class AssetSerializer(BulkOrgResourceModelSerializer): @classmethod def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('labels', 'nodes')\ - .select_related('admin_user') + queryset = queryset.prefetch_related('labels', 'nodes', 'protocols')\ + .select_related('admin_user', 'domain') return queryset @staticmethod diff --git a/apps/assets/serializers/system_user.py b/apps/assets/serializers/system_user.py index 0caffd2fcc48a9c7ac3ec0ed67b305c035b01d24..116dcdd362d1abd803acde8738a5c01a0e55eb4c 100644 --- a/apps/assets/serializers/system_user.py +++ b/apps/assets/serializers/system_user.py @@ -20,7 +20,7 @@ class SystemUserSerializer(BulkOrgResourceModelSerializer): 'id', 'name', 'username', 'password', 'public_key', 'private_key', 'login_mode', 'login_mode_display', 'priority', 'protocol', 'auto_push', 'cmd_filters', 'sudo', 'shell', 'comment', 'nodes', - 'assets', 'assets_amount', 'connectivity_amount' + 'assets_amount', 'connectivity_amount' ] extra_kwargs = { 'password': {"write_only": True}, @@ -32,6 +32,12 @@ class SystemUserSerializer(BulkOrgResourceModelSerializer): 'created_by': {'read_only': True}, } + @classmethod + def setup_eager_loading(cls, queryset): + """ Perform necessary eager loading of data. """ + queryset = queryset.prefetch_related('cmd_filters', 'nodes') + return queryset + class SystemUserAuthSerializer(AuthSerializer): """ @@ -47,8 +53,6 @@ class SystemUserAuthSerializer(AuthSerializer): - - class SystemUserSimpleSerializer(serializers.ModelSerializer): """ 系统用户最基本信息的数据结构 diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index a145437d0bd8c4c7ca2658f13369b07257cbb0dd..59d01c98a034a879f52172895c0c1699c0ed86e8 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -27,11 +27,6 @@ def test_asset_conn_on_created(asset): test_asset_connectivity_util.delay([asset]) -def set_asset_root_node(asset): - logger.debug("Set asset default node: {}".format(Node.root())) - asset.nodes.add(Node.root()) - - @receiver(post_save, sender=Asset, dispatch_uid="my_unique_identifier") @on_transaction_commit def on_asset_created_or_update(sender, instance=None, created=False, **kwargs): diff --git a/apps/assets/templates/assets/admin_user_list.html b/apps/assets/templates/assets/admin_user_list.html index 395e8d693d938414feecfd5f3fbfec8fa0507ac2..a930a6ce0eab93e2b6b4e21ed7956ba470c8af04 100644 --- a/apps/assets/templates/assets/admin_user_list.html +++ b/apps/assets/templates/assets/admin_user_list.html @@ -84,7 +84,7 @@ function initTable() { $(td).html(innerHtml) }}, {targets: 5, createdCell: function (td, cellData) { - var data = cellData['unreachable']; + var data = cellData.unreachable; var innerHtml = ""; if (data !== 0) { innerHtml = "" + data + ""; diff --git a/apps/assets/templates/assets/system_user_list.html b/apps/assets/templates/assets/system_user_list.html index a30d25a519f3fb0f68fbf3e30957d36b4313bd4a..680bfa421aebea5bc35b7f1bd2aa7262a3f9c44e 100644 --- a/apps/assets/templates/assets/system_user_list.html +++ b/apps/assets/templates/assets/system_user_list.html @@ -80,7 +80,7 @@ function initTable() { }}, {targets: 6, createdCell: function (td, cellData) { var innerHtml = ""; - var data = cellData['reachable']; + var data = cellData.reachable; if (data !== 0) { innerHtml = "" + data + ""; } else { @@ -89,7 +89,7 @@ function initTable() { $(td).html(innerHtml) }}, {targets: 7, createdCell: function (td, cellData) { - var data = cellData['unreachable']; + var data = cellData.unreachable; var innerHtml = ""; if (data !== 0) { innerHtml = "" + data + ""; @@ -103,7 +103,7 @@ function initTable() { var innerHtml = ""; var total = rowData.assets_amount; var reachable = cellData.reachable; - if (total !== 0) { + if (total && total !== 0) { val = reachable/total * 100; } @@ -119,7 +119,8 @@ function initTable() { var update_btn = '{% trans "Update" %}'.replace('{{ DEFAULT_PK }}', cellData); var del_btn = '{% trans "Delete" %}'.replace('{{ DEFAULT_PK }}', cellData); $(td).html(update_btn + del_btn) - }}], + }}, + ], ajax_url: '{% url "api-assets:system-user-list" %}', columns: [ {data: "id" }, {data: "name" }, {data: "username" }, {data: "protocol"}, {data: "login_mode_display"}, {data: "assets_amount" }, diff --git a/apps/assets/views/asset.py b/apps/assets/views/asset.py index 984e2052e43ef0b36ed2a2a1db823817d4804a92..ed0fe7efcc9292856975d43776dd756d5d4eebb5 100644 --- a/apps/assets/views/asset.py +++ b/apps/assets/views/asset.py @@ -220,6 +220,11 @@ class AssetDetailView(PermissionsMixin, DetailView): template_name = 'assets/asset_detail.html' permission_classes = [IsValidUser] + def get_queryset(self): + return super().get_queryset().prefetch_related( + "nodes", "labels", "protocols" + ).select_related('admin_user', 'domain') + def get_context_data(self, **kwargs): nodes_remain = Node.objects.exclude(assets=self.object) context = { diff --git a/apps/assets/views/system_user.py b/apps/assets/views/system_user.py index b7976ed2ff40b6b99c01700ccab30f8b3effdfa8..ca363ac032f6bdc5e5a3d4f3be9d0e3433fdb332 100644 --- a/apps/assets/views/system_user.py +++ b/apps/assets/views/system_user.py @@ -97,7 +97,10 @@ class SystemUserAssetView(PermissionsMixin, DetailView): permission_classes = [IsOrgAdmin] def get_context_data(self, **kwargs): - nodes_remain = sorted(Node.objects.exclude(systemuser=self.object), reverse=True) + from ..utils import NodeUtil + nodes_remain = Node.objects.exclude(systemuser=self.object) + util = NodeUtil() + nodes_remain = util.get_nodes_by_queryset(nodes_remain) context = { 'app': _('assets'), 'action': _('System user asset'), diff --git a/apps/common/signals_handlers.py b/apps/common/signals_handlers.py index cf8cc9a787f19eae50606951c15e7eac62d0db5d..8a846300638373048b956b2e467c0508d732fbcd 100644 --- a/apps/common/signals_handlers.py +++ b/apps/common/signals_handlers.py @@ -8,11 +8,12 @@ from django.core.signals import request_finished from django.db import connection -from .utils import get_logger +from common.utils import get_logger from .local import thread_local -logger = get_logger(__file__) pattern = re.compile(r'FROM `(\w+)`') +# logger = logging.getLogger('jmsdb') +logger = get_logger(__name__) class Counter: diff --git a/apps/orgs/mixins/api.py b/apps/orgs/mixins/api.py index 5e0cef5c3959b768441be7cfbabe3a39879818bf..180e9770bcfd6e4048f9e64d9ef8b80542a757f7 100644 --- a/apps/orgs/mixins/api.py +++ b/apps/orgs/mixins/api.py @@ -27,7 +27,11 @@ class OrgModelViewSet(IDInCacheFilterMixin, ModelViewSet): class OrgBulkModelViewSet(IDInCacheFilterMixin, BulkModelViewSet): def get_queryset(self): - return super().get_queryset().all() + queryset = super().get_queryset().all() + if hasattr(self, 'serializer_class') and \ + hasattr(self.serializer_class, 'setup_eager_loading'): + queryset = self.serializer_class.setup_eager_loading(queryset) + return queryset class OrgMembershipModelViewSetMixin: diff --git a/apps/terminal/api/session.py b/apps/terminal/api/session.py index e81772fae6dd82f120566b655ff4f7bc9bcd6763..6dd48f591636f4edf657cc19356819fa601a757a 100644 --- a/apps/terminal/api/session.py +++ b/apps/terminal/api/session.py @@ -31,7 +31,8 @@ class SessionViewSet(OrgBulkModelViewSet): pagination_class = LimitOffsetPagination permission_classes = (IsOrgAdminOrAppUser | IsAuditor, ) filter_fields = [ - "user", "asset", "system_user", "terminal", "is_finished", + "user", "asset", "system_user", "remote_addr", + "protocol", "terminal", "is_finished", ] date_range_filter_fields = [ ('date_start', ('date_from', 'date_to')) diff --git a/apps/terminal/models.py b/apps/terminal/models.py index 2c5c7d66672af8b66b5c09fc9a65d57709c04d82..43b8c19ce6bc0fe9988e17cdd47e8eb291983e8b 100644 --- a/apps/terminal/models.py +++ b/apps/terminal/models.py @@ -191,26 +191,22 @@ class Session(OrgModelMixin): @property def _date_start_first_has_replay_rdp_session(self): - if self._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION is None: + if self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION is None: instance = self.__class__.objects.filter( - protocol='rdp', has_replay=True).order_by('date_start').first() + protocol='rdp', has_replay=True + ).order_by('date_start').first() if not instance: - return None - self._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION = instance.date_start - - return self._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION + date_start = timezone.now() - timezone.timedelta(days=365) + else: + date_start = instance.date_start + self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION = date_start + return self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION def can_replay(self): if self.has_replay: return True - - # 判断对RDP Session添加上报has_replay状态机制之前的录像回放 - if self._date_start_first_has_replay_rdp_session is None: - return True - if self.date_start < self._date_start_first_has_replay_rdp_session: return True - return False def save_to_storage(self, f): diff --git a/apps/terminal/templates/terminal/command_list.html b/apps/terminal/templates/terminal/command_list.html index 751e63ac91b0ef6bcb884b6c2ce58286d81becd0..3c999f0f6ee01159e7a078b91d59f5d1b45c3845 100644 --- a/apps/terminal/templates/terminal/command_list.html +++ b/apps/terminal/templates/terminal/command_list.html @@ -71,7 +71,7 @@ {% block content_bottom_left %}{% endblock %} {% block custom_foot_js %} - + +