[Update] Debug cache

apps/assets/templates/assets/user_asset_list.html

@@ -151,7 +151,7 @@ function initTree() {
 }
 
 $(document).ready(function () {
-    initTree();
+    {#initTree();#}
     initTable();
 }).on('click', '.labels li', function () {
     var val = $(this).text();

apps/perms/api/__init__.py

@@ -5,3 +5,4 @@ from .asset_permission import *
 from .user_permission import *
 from .user_group_permission import *
 from .remote_app_permission import *
+from .user_remote_app_permission import *

apps/perms/api/user_permission.py

 # -*- coding: utf-8 -*-
 #
 import time
+import traceback
 from hashlib import md5
 from django.core.cache import cache
 from django.conf import settings
@@ -16,16 +17,12 @@ from common.tree import TreeNodeSerializer
 from common.utils import get_logger
 from ..utils import (
     AssetPermissionUtil, parse_asset_to_tree_node, parse_node_to_tree_node,
-    check_system_user_action, RemoteAppPermissionUtil,
-    construct_remote_apps_tree_root, parse_remote_app_to_tree_node,
-)
-from ..hands import (
-    User, Asset, Node, SystemUser, RemoteApp,
-    NodeSerializer, RemoteAppSerializer,
+    check_system_user_action,
 )
+from ..hands import User, Asset, Node, SystemUser, NodeSerializer
 from .. import serializers, const
 from ..mixins import (
-    AssetsFilterMixin, RemoteAppFilterMixin
+    AssetsFilterMixin,
 )
 from ..models import Action
 
@@ -36,8 +33,6 @@ __all__ = [
     'UserGrantedNodesWithAssetsApi', 'UserGrantedNodeAssetsApi',
     'ValidateUserAssetPermissionApi', 'UserGrantedNodeChildrenApi',
     'UserGrantedNodesWithAssetsAsTreeApi', 'GetUserAssetPermissionActionsApi',
-    'UserGrantedRemoteAppsApi', 'ValidateUserRemoteAppPermissionApi',
-    'UserGrantedRemoteAppsAsTreeApi',
 ]
 
 
@@ -77,46 +72,61 @@ class UserPermissionCacheMixin:
         request_md5 = self.get_request_md5()
         meta_cache_id = self.get_meta_cache_id()
         resp_cache_id = '{}_{}_{}'.format(obj_id, request_md5, meta_cache_id)
+        resp_cache_id = md5(resp_cache_id.encode()).hexdigest()
         return resp_cache_id
 
     def get_response_from_cache(self):
-        resp_cache_id = self.get_response_cache_id()
         # 没有数据缓冲
         meta_cache_id = self.get_meta_cache_id()
         if not meta_cache_id:
+            logger.debug("Not get meta id: {}".format(meta_cache_id))
             return None
         # 从响应缓冲里获取响应
-        key = self.RESP_CACHE_KEY.format(resp_cache_id)
+        key = self.get_response_key()
+        print("response key: {}".format(key))
         data = cache.get(key)
+        print(data)
         if not data:
+            logger.debug("Not get response from cache: {}".format(key))
             return None
         logger.debug("Get user permission from cache: {}".format(self.get_object()))
         response = Response(data)
         return response
 
     def expire_response_cache(self):
+        print("Expire cache")
         obj_id = self.get_object_id()
         expire_cache_id = '{}_{}'.format(obj_id, '*')
         key = self.RESP_CACHE_KEY.format(expire_cache_id)
         cache.delete_pattern(key)
 
-    def set_response_to_cache(self, response):
+    def get_response_key(self):
         resp_cache_id = self.get_response_cache_id()
         key = self.RESP_CACHE_KEY.format(resp_cache_id)
+        return key
+
+    def set_response_to_cache(self, response):
+        key = self.get_response_key()
         cache.set(key, response.data, self.CACHE_TIME)
+        logger.debug("Set response to cache: {}".format(key))
+        print(self.CACHE_TIME)
 
     def get(self, request, *args, **kwargs):
         self.cache_policy = request.GET.get('cache_policy', '0')
 
         obj = self._get_object()
         if obj is None:
+            logger.debug("Not get response from cache: obj is none")
             return super().get(request, *args, **kwargs)
 
         if AssetPermissionUtil.is_not_using_cache(self.cache_policy):
+            logger.debug("Not get resp from cache: {}".format(self.cache_policy))
             return super().get(request, *args, **kwargs)
         elif AssetPermissionUtil.is_refresh_cache(self.cache_policy):
+            logger.debug("Not get resp from cache: {}".format(self.cache_policy))
             self.expire_response_cache()
 
+        logger.debug("Try get response from cache")
         resp = self.get_response_from_cache()
         if not resp:
             resp = super().get(request, *args, **kwargs)
@@ -245,11 +255,8 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionCacheMixin, ListAPIView)
             user = get_object_or_404(User, id=user_id)
         return user
 
-    def list(self, request, *args, **kwargs):
-        resp = super().list(request, *args, **kwargs)
-        return resp
-
     def get_queryset(self):
+        print("Call get queryset")
         queryset = []
         self.show_assets = self.request.query_params.get('show_assets', '1') == '1'
         self.system_user_id = self.request.query_params.get('system_user')
@@ -259,22 +266,14 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionCacheMixin, ListAPIView)
             util.filter_permissions(
                 system_users=self.system_user_id
             )
-        print("111111111111")
         nodes = util.get_nodes_with_assets()
-        print("22222222222222")
         for node, assets in nodes.items():
-            now = time.time()
-            print("Parse to node")
             data = parse_node_to_tree_node(node)
-            print("parse to node end, using: {0:.2f}".format(time.time() - now))
             queryset.append(data)
             if not self.show_assets:
                 continue
             for asset, system_users in assets.items():
-                now1 = time.time()
-                print("parse to asset")
                 data = parse_asset_to_tree_node(node, asset, system_users)
-                print("parse to asset end, using: {0:.2f}".format(time.time()-now1))
                 queryset.append(data)
         return queryset
 
@@ -458,77 +457,3 @@ class GetUserAssetPermissionActionsApi(UserPermissionCacheMixin, APIView):
         return Response({'actions': actions}, status=200)
 
 
-# RemoteApp permission
-
-class UserGrantedRemoteAppsApi(RemoteAppFilterMixin, ListAPIView):
-    permission_classes = (IsOrgAdminOrAppUser,)
-    serializer_class = RemoteAppSerializer
-    pagination_class = LimitOffsetPagination
-
-    def get_object(self):
-        user_id = self.kwargs.get('pk', '')
-        if user_id:
-            user = get_object_or_404(User, id=user_id)
-        else:
-            user = self.request.user
-        return user
-
-    def get_queryset(self):
-        util = RemoteAppPermissionUtil(self.get_object())
-        queryset = util.get_remote_apps()
-        queryset = list(queryset)
-        return queryset
-
-    def get_permissions(self):
-        if self.kwargs.get('pk') is None:
-            self.permission_classes = (IsValidUser,)
-        return super().get_permissions()
-
-
-class UserGrantedRemoteAppsAsTreeApi(ListAPIView):
-    serializer_class = TreeNodeSerializer
-    permission_classes = (IsOrgAdminOrAppUser,)
-
-    def get_object(self):
-        user_id = self.kwargs.get('pk', '')
-        if not user_id:
-            user = self.request.user
-        else:
-            user = get_object_or_404(User, id=user_id)
-        return user
-
-    def get_queryset(self):
-        queryset = []
-        tree_root = construct_remote_apps_tree_root()
-        queryset.append(tree_root)
-
-        util = RemoteAppPermissionUtil(self.get_object())
-        remote_apps = util.get_remote_apps()
-        for remote_app in remote_apps:
-            node = parse_remote_app_to_tree_node(tree_root, remote_app)
-            queryset.append(node)
-
-        queryset = sorted(queryset)
-        return queryset
-
-    def get_permissions(self):
-        if self.kwargs.get('pk') is None:
-            self.permission_classes = (IsValidUser,)
-        return super().get_permissions()
-
-
-class ValidateUserRemoteAppPermissionApi(APIView):
-    permission_classes = (IsOrgAdminOrAppUser,)
-
-    def get(self, request, *args, **kwargs):
-        self.change_org_if_need(request, kwargs)
-        user_id = request.query_params.get('user_id', '')
-        remote_app_id = request.query_params.get('remote_app_id', '')
-        user = get_object_or_404(User, id=user_id)
-        remote_app = get_object_or_404(RemoteApp, id=remote_app_id)
-
-        util = RemoteAppPermissionUtil(user)
-        remote_apps = util.get_remote_apps()
-        if remote_app not in remote_apps:
-            return Response({'msg': False}, status=403)
-        return Response({'msg': True}, status=200)

apps/perms/api/user_remote_app_permission.py

+# -*- coding: utf-8 -*-
+
+from django.shortcuts import get_object_or_404
+from rest_framework.views import APIView, Response
+from rest_framework.generics import (
+    ListAPIView, get_object_or_404,
+)
+from rest_framework.pagination import LimitOffsetPagination
+
+from common.permissions import IsValidUser, IsOrgAdminOrAppUser
+from common.tree import TreeNodeSerializer
+from ..utils import (
+    RemoteAppPermissionUtil, construct_remote_apps_tree_root,
+    parse_remote_app_to_tree_node,
+)
+from ..hands import User, RemoteApp, RemoteAppSerializer
+from ..mixins import RemoteAppFilterMixin
+
+
+__all__ = [
+    'UserGrantedRemoteAppsApi', 'ValidateUserRemoteAppPermissionApi',
+    'UserGrantedRemoteAppsAsTreeApi',
+]
+
+
+class UserGrantedRemoteAppsApi(RemoteAppFilterMixin, ListAPIView):
+    permission_classes = (IsOrgAdminOrAppUser,)
+    serializer_class = RemoteAppSerializer
+    pagination_class = LimitOffsetPagination
+
+    def get_object(self):
+        user_id = self.kwargs.get('pk', '')
+        if user_id:
+            user = get_object_or_404(User, id=user_id)
+        else:
+            user = self.request.user
+        return user
+
+    def get_queryset(self):
+        util = RemoteAppPermissionUtil(self.get_object())
+        queryset = util.get_remote_apps()
+        queryset = list(queryset)
+        return queryset
+
+    def get_permissions(self):
+        if self.kwargs.get('pk') is None:
+            self.permission_classes = (IsValidUser,)
+        return super().get_permissions()
+
+
+class UserGrantedRemoteAppsAsTreeApi(ListAPIView):
+    serializer_class = TreeNodeSerializer
+    permission_classes = (IsOrgAdminOrAppUser,)
+
+    def get_object(self):
+        user_id = self.kwargs.get('pk', '')
+        if not user_id:
+            user = self.request.user
+        else:
+            user = get_object_or_404(User, id=user_id)
+        return user
+
+    def get_queryset(self):
+        queryset = []
+        tree_root = construct_remote_apps_tree_root()
+        queryset.append(tree_root)
+
+        util = RemoteAppPermissionUtil(self.get_object())
+        remote_apps = util.get_remote_apps()
+        for remote_app in remote_apps:
+            node = parse_remote_app_to_tree_node(tree_root, remote_app)
+            queryset.append(node)
+
+        queryset = sorted(queryset)
+        return queryset
+
+    def get_permissions(self):
+        if self.kwargs.get('pk') is None:
+            self.permission_classes = (IsValidUser,)
+        return super().get_permissions()
+
+
+class ValidateUserRemoteAppPermissionApi(APIView):
+    permission_classes = (IsOrgAdminOrAppUser,)
+
+    def get(self, request, *args, **kwargs):
+        user_id = request.query_params.get('user_id', '')
+        remote_app_id = request.query_params.get('remote_app_id', '')
+        user = get_object_or_404(User, id=user_id)
+        remote_app = get_object_or_404(RemoteApp, id=remote_app_id)
+
+        util = RemoteAppPermissionUtil(user)
+        remote_apps = util.get_remote_apps()
+        if remote_app not in remote_apps:
+            return Response({'msg': False}, status=403)
+        return Response({'msg': True}, status=200)

apps/perms/utils/asset_permission.py

@@ -110,17 +110,17 @@ class GenerateTree:
         self._ungroup_node = node
         return node
 
-    @timeit
+    #@timeit
     def add_assets_without_system_users(self, assets):
         for asset in assets:
             self.add_asset(asset, {})
 
-    @timeit
+    #@timeit
     def add_assets(self, assets):
         for asset, system_users in assets.items():
             self.add_asset(asset, system_users)
 
-    @timeit
+    #@timeit
     def add_asset(self, asset, system_users=None):
         nodes = asset.nodes.all()
         nodes = self.node_util.get_nodes_by_queryset(nodes)
@@ -159,7 +159,7 @@ class GenerateTree:
         self.nodes[node]["system_users"] = system_users
 
     # 添加树节点
-    @timeit
+    #@timeit
     def add_nodes(self, nodes):
         _nodes = nodes.keys()
         family = self.node_util.get_family(_nodes, with_children=True)
@@ -169,7 +169,7 @@ class GenerateTree:
     def get_assets(self):
         return dict(self.assets)
 
-    @timeit
+    #@timeit
     def get_nodes_with_assets(self):
         if self._nodes_with_assets:
             return self._nodes_with_assets
@@ -220,6 +220,7 @@ class AssetPermissionCacheMixin:
     CACHE_META_KEY_PREFIX = '_ASSET_PERM_META_KEY_'
     CACHE_TIME = settings.ASSETS_PERM_CACHE_TIME
     CACHE_POLICY_MAP = (('0', 'never'), ('1', 'using'), ('2', 'refresh'))
+    cache_policy = '1'
 
     @classmethod
     def is_not_using_cache(cls, cache_policy):
@@ -242,7 +243,7 @@ class AssetPermissionCacheMixin:
     def _is_refresh_cache(self):
         return self.is_refresh_cache(self.cache_policy)
 
-    @timeit
+    #@timeit
     def get_cache_key(self, resource):
         cache_key = self.CACHE_KEY_PREFIX + '{obj_id}_{filter_id}_{resource}'
         return cache_key.format(
@@ -250,18 +251,23 @@ class AssetPermissionCacheMixin:
             resource=resource
         )
 
+    @property
     def node_key(self):
         return self.get_cache_key('NODES_WITH_ASSETS')
 
+    @property
     def asset_key(self):
-        return self.get_cache_key('ASSETS')
+        key = self.get_cache_key('ASSETS')
+        return key
 
+    @property
     def system_key(self):
         return self.get_cache_key('SYSTEM_USER')
 
     def get_assets_from_cache(self):
         cached = cache.get(self.asset_key)
         if not cached:
+            print("Refresh cache")
             self.update_cache()
             cached = cache.get(self.asset_key)
         return cached
@@ -290,7 +296,7 @@ class AssetPermissionCacheMixin:
         return cached
 
     def get_assets(self):
-        if self._is_not_using_cache():
+        if self._is_using_cache():
             return self.get_assets_from_cache()
         elif self._is_refresh_cache():
             self.expire_cache()
@@ -318,7 +324,10 @@ class AssetPermissionCacheMixin:
     @property
     def cache_meta(self):
         key = self.get_meta_cache_key()
-        return cache.get(key) or {}
+        meta = cache.get(key) or {}
+        # print("Meta key: {}".format(key))
+        # print("Meta id: {}".format(meta["id"]))
+        return meta
 
     def set_meta_to_cache(self):
         key = self.get_meta_cache_key()
@@ -327,6 +336,8 @@ class AssetPermissionCacheMixin:
             'datetime': timezone.now(),
             'object': str(self.object)
         }
+        # print("Set meta key: {}".format(key))
+        # print("set meta to cache: {}".format(meta["id"]))
         cache.set(key, meta, self.CACHE_TIME)
 
     def expire_cache_meta(self):
@@ -403,13 +414,13 @@ class AssetPermissionUtil(AssetPermissionCacheMixin):
         self._permissions = permissions
         return permissions
 
-    @timeit
+    #@timeit
     def filter_permissions(self, **filters):
         filters_json = json.dumps(filters, sort_keys=True)
         self._permissions = self.permissions.filter(**filters)
         self._filter_id = md5(filters_json.encode()).hexdigest()
 
-    @timeit
+    #@timeit
     def get_nodes_direct(self):
         """
         返回用户/组授权规则直接关联的节点
@@ -425,7 +436,7 @@ class AssetPermissionUtil(AssetPermissionCacheMixin):
         self.tree.add_nodes(nodes)
         return nodes
 
-    @timeit
+    #@timeit
     def get_assets_direct(self):
         """
         返回用户授权规则直接关联的资产
@@ -442,7 +453,7 @@ class AssetPermissionUtil(AssetPermissionCacheMixin):
         self.tree.add_assets(assets)
         return assets
 
-    @timeit
+    #@timeit
     def get_assets_without_cache(self):
         """
         :return: {asset1: set(system_user1,)}
@@ -469,7 +480,7 @@ class AssetPermissionUtil(AssetPermissionCacheMixin):
         self._assets = assets
         return assets
 
-    @timeit
+    #@timeit
     def get_nodes_with_assets_without_cache(self):
         """
         返回节点并且包含资产

apps/users/templates/users/user_granted_asset.html

@@ -145,7 +145,7 @@ function initTree() {
 
 $(document).ready(function () {
     initTree();
-    initTable();
+    {#initTable();#}
 });
 </script>
 {% endblock %}