Merge pull request #3836 from jumpserver/1.5.8_cap

1.5.8 cap

apps/assets/backends/manager.py

@@ -154,8 +154,8 @@ class AssetUserManager:
 
     @staticmethod
     def create(**kwargs):
-        authbook = AuthBook(**kwargs)
-        authbook.save()
+        # 使用create方法创建AuthBook对象，解决并发创建问题（添加锁机制）
+        authbook = AuthBook.create(**kwargs)
         return authbook
 
     def __getattr__(self, item):

apps/assets/models/authbook.py

 # -*- coding: utf-8 -*-
 #
 
-from django.db import models
+from django.db import models, transaction
+from django.db.models import Max
+from django.core.cache import cache
 from django.utils.translation import ugettext_lazy as _
 
 from orgs.mixins.models import OrgManager
@@ -11,8 +13,8 @@ __all__ = ['AuthBook']
 
 
 class AuthBookQuerySet(models.QuerySet):
-    def latest_version(self):
-        return self.filter(is_latest=True)
+    def delete(self):
+        raise PermissionError("Bulk delete authbook deny")
 
 
 class AuthBookManager(OrgManager):
@@ -33,37 +35,42 @@ class AuthBook(BaseUser):
     class Meta:
         verbose_name = _('AuthBook')
 
-    def set_to_latest(self):
-        self.remove_pre_latest()
-        self.is_latest = True
-        self.save()
-
-    def get_pre_latest(self):
-        pre_obj = self.__class__.objects.filter(
-            username=self.username, asset=self.asset
-        ).latest_version().first()
-        return pre_obj
-
-    def remove_pre_latest(self):
-        pre_obj = self.get_pre_latest()
-        if pre_obj:
-            pre_obj.is_latest = False
-            pre_obj.save()
-
-    def set_version(self):
-        pre_obj = self.get_pre_latest()
-        if pre_obj:
-            self.version = pre_obj.version + 1
-        else:
-            self.version = 1
-        self.save()
-
     def get_related_assets(self):
         return [self.asset]
 
     def generate_id_with_asset(self, asset):
         return self.id
 
+    @classmethod
+    def get_max_version(cls, username, asset):
+        version_max = cls.objects.filter(username=username, asset=asset) \
+            .aggregate(Max('version'))
+        version_max = version_max['version__max'] or 0
+        return version_max
+
+    @classmethod
+    def create(cls, **kwargs):
+        """
+        使用并发锁机制创建AuthBook对象, (主要针对并发创建 username, asset 相同的对象时)
+        并更新其他对象的 is_latest=False (其他对象: 与当前对象的 username, asset 相同)
+        同时设置自己的 is_latest=True, version=max_version + 1
+        """
+        username = kwargs['username']
+        asset = kwargs['asset']
+        key_lock = 'KEY_LOCK_CREATE_AUTH_BOOK_{}_{}'.format(username, asset.id)
+        with cache.lock(key_lock):
+            with transaction.atomic():
+                cls.objects.filter(
+                    username=username, asset=asset, is_latest=True
+                ).update(is_latest=False)
+                max_version = cls.get_max_version(username, asset)
+                kwargs.update({
+                    'version': max_version + 1,
+                    'is_latest': True
+                })
+                obj = cls.objects.create(**kwargs)
+                return obj
+
     @property
     def connectivity(self):
         return self.get_asset_connectivity(self.asset)

apps/assets/models/node.py

@@ -574,14 +574,13 @@ class Node(OrgModelMixin, SomeNodesMixin, TreeMixin, FamilyMixin, FullValueMixin
         org = get_current_org()
         if not org or not org.is_real():
             Organization.default().change_to()
-        i = 0
-        while i < count:
-            nodes = list(cls.objects.all())
-            if count > 100:
-                length = 100
-            else:
-                length = count
-
-            for i in range(length):
-                node = random.choice(nodes)
-                node.create_child('Node {}'.format(i))
+        nodes = list(cls.objects.all())
+        if count > 100:
+            length = 100
+        else:
+            length = count
+
+        for i in range(length):
+            node = random.choice(nodes)
+            child = node.create_child('Node {}'.format(i))
+            print("{}. {}".format(i, child))

apps/assets/serializers/asset_user.py

@@ -37,7 +37,6 @@ class AssetUserWriteSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializ
         if not validated_data.get("name") and validated_data.get("username"):
             validated_data["name"] = validated_data["username"]
         instance = AssetUserManager.create(**validated_data)
-        instance.set_to_latest()
         return instance
 
 

apps/assets/signals_handler.py

@@ -15,7 +15,6 @@ from .utils import TreeService
 from .tasks import (
     update_assets_hardware_info_util,
     test_asset_connectivity_util,
-    push_system_user_to_assets,
     push_system_user_to_assets_manual,
     push_system_user_to_assets,
     add_nodes_assets_to_system_users
@@ -235,9 +234,3 @@ def on_node_update_or_created(sender, **kwargs):
     Node.refresh_nodes()
     with tmp_to_root_org():
         Node.refresh_nodes()
-
-
-@receiver(post_save, sender=AuthBook)
-def on_authbook_created(sender, instance=None, created=True, **kwargs):
-    if created and instance:
-        instance.set_version()

apps/jumpserver/settings/base.py

@@ -232,7 +232,8 @@ FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o755
 # Cache use redis
 CACHES = {
     'default': {
-        'BACKEND': 'redis_cache.RedisCache',
+        # 'BACKEND': 'redis_cache.RedisCache',
+        'BACKEND': 'redis_lock.django_cache.RedisCache',
         'LOCATION': 'redis://:%(password)s@%(host)s:%(port)s/%(db)s' % {
             'password': CONFIG.REDIS_PASSWORD,
             'host': CONFIG.REDIS_HOST,

apps/jumpserver/views/index.py

@@ -24,7 +24,8 @@ class MonthLoginMetricMixin:
 
     @lazyproperty
     def session_month_dates(self):
-        return self.session_month.dates('date_start', 'day')
+        dates = self.session_month.dates('date_start', 'day')
+        return dates
 
     def get_month_day_metrics(self):
         month_str = [
@@ -57,12 +58,22 @@ class MonthLoginMetricMixin:
     def asset_disabled_total(self):
         return Asset.objects.filter(is_active=False).count()
 
+    @staticmethod
+    def get_date_start_2_end(d):
+        time_min = timezone.datetime.min.time()
+        time_max = timezone.datetime.max.time()
+        tz = timezone.get_current_timezone()
+        ds = timezone.datetime.combine(d, time_min).replace(tzinfo=tz)
+        de = timezone.datetime.combine(d, time_max).replace(tzinfo=tz)
+        return ds, de
+
     def get_date_login_count(self, date):
         tp = "LOGIN"
         count = self.__get_data_from_cache(date, tp)
         if count is not None:
             return count
-        count = Session.objects.filter(date_start__date=date).count()
+        ds, de = self.get_date_start_2_end(date)
+        count = Session.objects.filter(date_start__range=(ds, de)).count()
         self.__set_data_to_cache(date, tp, count)
         return count
 
@@ -80,7 +91,8 @@ class MonthLoginMetricMixin:
         count = self.__get_data_from_cache(date, tp)
         if count is not None:
             return count
-        count = Session.objects.filter(date_start__date=date)\
+        ds, de = self.get_date_start_2_end(date)
+        count = Session.objects.filter(date_start__range=(ds, de))\
             .values('user').distinct().count()
         self.__set_data_to_cache(date, tp, count)
         return count
@@ -97,7 +109,8 @@ class MonthLoginMetricMixin:
         count = self.__get_data_from_cache(date, tp)
         if count is not None:
             return count
-        count = Session.objects.filter(date_start__date=date) \
+        ds, de = self.get_date_start_2_end(date)
+        count = Session.objects.filter(date_start__range=(ds, de)) \
             .values('asset').distinct().count()
         self.__set_data_to_cache(date, tp, count)
         return count

apps/locale/zh/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: JumpServer 0.3.3\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-03-18 17:58+0800\n"
+"POT-Creation-Date: 2020-03-23 03:05+0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: ibuler <ibuler@qq.com>\n"
 "Language-Team: JumpServer team<ibuler@qq.com>\n"
@@ -3120,23 +3120,23 @@ msgstr "Become"
 msgid "Create by"
 msgstr "创建者"
 
-#: ops/models/adhoc.py:232
+#: ops/models/adhoc.py:233
 msgid "Task display"
 msgstr "任务展示"
 
-#: ops/models/adhoc.py:233
+#: ops/models/adhoc.py:234
 msgid "Host amount"
 msgstr "主机数量"
 
-#: ops/models/adhoc.py:235
+#: ops/models/adhoc.py:236
 msgid "Start time"
 msgstr "开始时间"
 
-#: ops/models/adhoc.py:236
+#: ops/models/adhoc.py:237
 msgid "End time"
 msgstr "完成时间"
 
-#: ops/models/adhoc.py:237 ops/templates/ops/adhoc_history.html:55
+#: ops/models/adhoc.py:238 ops/templates/ops/adhoc_history.html:55
 #: ops/templates/ops/task_history.html:61 ops/templates/ops/task_list.html:16
 #: xpack/plugins/change_auth_plan/models.py:172
 #: xpack/plugins/change_auth_plan/models.py:294
@@ -3146,31 +3146,31 @@ msgstr "完成时间"
 msgid "Time"
 msgstr "时间"
 
-#: ops/models/adhoc.py:238 ops/templates/ops/adhoc_detail.html:104
+#: ops/models/adhoc.py:239 ops/templates/ops/adhoc_detail.html:104
 #: ops/templates/ops/adhoc_history.html:53
 #: ops/templates/ops/adhoc_history_detail.html:67
 #: ops/templates/ops/task_detail.html:82 ops/templates/ops/task_history.html:59
 msgid "Is finished"
 msgstr "是否完成"
 
-#: ops/models/adhoc.py:239 ops/templates/ops/adhoc_history.html:54
+#: ops/models/adhoc.py:240 ops/templates/ops/adhoc_history.html:54
 #: ops/templates/ops/task_history.html:60
 msgid "Is success"
 msgstr "是否成功"
 
-#: ops/models/adhoc.py:240
+#: ops/models/adhoc.py:241
 msgid "Adhoc raw result"
 msgstr "结果"
 
-#: ops/models/adhoc.py:241
+#: ops/models/adhoc.py:242
 msgid "Adhoc result summary"
 msgstr "汇总"
 
-#: ops/models/adhoc.py:281 xpack/plugins/change_auth_plan/utils.py:86
+#: ops/models/adhoc.py:282 xpack/plugins/change_auth_plan/utils.py:89
 msgid "{} Start task: {}"
 msgstr "{} 任务开始: {}"
 
-#: ops/models/adhoc.py:290 xpack/plugins/change_auth_plan/utils.py:98
+#: ops/models/adhoc.py:291 xpack/plugins/change_auth_plan/utils.py:101
 msgid "{} Task finish"
 msgstr "{} 任务结束"
 
@@ -6275,6 +6275,10 @@ msgstr "步骤"
 msgid "Change auth plan task"
 msgstr "改密计划任务"
 
+#: xpack/plugins/change_auth_plan/serializers.py:58
+msgid "* For security, do not change {}'s password"
+msgstr "* 为了安全，不能修改 {} 的密码"
+
 #: xpack/plugins/change_auth_plan/serializers.py:68
 msgid "* Please enter custom password"
 msgstr "* 请输入自定义密码"
@@ -6344,11 +6348,11 @@ msgstr "执行失败"
 msgid "Create plan"
 msgstr "创建计划"
 
-#: xpack/plugins/change_auth_plan/utils.py:237
+#: xpack/plugins/change_auth_plan/utils.py:262
 msgid "Failed to connect asset"
 msgstr "连接资产失败"
 
-#: xpack/plugins/change_auth_plan/utils.py:239
+#: xpack/plugins/change_auth_plan/utils.py:264
 msgid "Incorrect password"
 msgstr "密码错误"
 

apps/ops/ansible/callback.py

@@ -132,6 +132,9 @@ class AdHocResultCallback(CallbackMixin, CallbackModule, CMDCallBackModule):
     def display_failed_stderr(self):
         pass
 
+    def set_play_context(self, context):
+        context.ssh_args = '-C -o ControlMaster=no'
+
 
 class CommandResultCallback(AdHocResultCallback):
     """

apps/ops/models/adhoc.py

@@ -278,7 +278,7 @@ class AdHocExecution(OrgModelMixin):
         raw = ''
 
         try:
-            date_start_s = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            date_start_s = timezone.now().now().strftime('%Y-%m-%d %H:%M:%S')
             print(_("{} Start task: {}").format(date_start_s, self.task.name))
             raw, summary = self.start_runner()
         except Exception as e:
@@ -286,7 +286,7 @@ class AdHocExecution(OrgModelMixin):
             raw = {"dark": {"all": str(e)}, "contacted": []}
         finally:
             self.clean_up(summary, time_start)
-            date_end = timezone.now()
+            date_end = timezone.now().now()
             date_end_s = date_end.strftime('%Y-%m-%d %H:%M:%S')
             print(_("{} Task finish").format(date_end_s))
             print('.\n\n.')

apps/terminal/templates/terminal/session_list.html

@@ -240,7 +240,7 @@ $(document).ready(function() {
     var hasConfirm = getCookie('replayConfirm');
     if (!hasConfirm) {
         var help_text = "{% trans "Visit doc for replay play offline: " %}";
-        help_text += "http://docs.jumpserver.org";
+        help_text += "https://github.com/jumpserver/videoplayer";
         var r = confirm(help_text);
         setCookie("replayConfirm", "1")
     }

requirements/requirements.txt

@@ -4,7 +4,7 @@ asn1crypto==0.24.0
 bcrypt==3.1.4
 billiard==3.5.0.3
 boto3==1.12.14
-botocore==1.9.5
+botocore==1.15.26
 celery==4.1.1
 certifi==2018.1.18
 cffi==1.13.2
@@ -61,8 +61,8 @@ pytz==2018.3
 PyYAML==5.1
 redis==2.10.6
 requests==2.22.0
-jms-storage==0.0.27
-s3transfer==0.1.13
+jms-storage==0.0.28
+s3transfer==0.3.3
 simplejson==3.13.2
 six==1.11.0
 sshpubkeys==3.1.0
@@ -96,3 +96,5 @@ django-cas-ng==4.0.1
 python-cas==1.5.0
 ipython
 huaweicloud-sdk-python==1.0.21
+django-redis==4.11.0
+python-redis-lock==3.5.0