From a822f667aff8c6bfd1b51eee98c4c73b9dd12b83 Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 9 Mar 2017 14:55:33 +0800 Subject: [PATCH] =?UTF-8?q?[Fixture]=20=E5=AE=8C=E6=88=90=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E6=8E=A8=E9=80=81task?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/forms.py | 69 +++-- apps/assets/models/asset.py | 2 +- apps/assets/models/user.py | 22 +- ...r_create_update.html => _system_user.html} | 22 +- .../templates/assets/system_user_create.html | 7 + .../templates/assets/system_user_update.html | 15 + apps/assets/utils.py | 2 + apps/assets/views.py | 9 +- apps/common/utils.py | 8 + apps/ops/models.py | 31 ++ apps/ops/models/__init__.py | 4 - apps/ops/models/ansible.py | 291 ------------------ apps/ops/models/task.py | 32 -- apps/ops/models/utils.py | 11 - apps/ops/tasks.py | 87 ++++++ apps/ops/tasks/__init__.py | 1 - apps/ops/tasks/_celery_tasks.py | 48 --- apps/ops/tasks/taskers.py | 126 -------- apps/ops/urls/view_urls.py | 5 +- apps/ops/utils/runner.py | 18 +- apps/ops/views.py | 24 +- apps/perms/hands.py | 5 - .../perms/asset_permission_create_update.html | 2 +- apps/perms/utils.py | 23 +- apps/perms/views.py | 6 + requirements/requirements.txt | 1 + run_server.py | 3 +- 27 files changed, 272 insertions(+), 602 deletions(-) rename apps/assets/templates/assets/{system_user_create_update.html => _system_user.html} (87%) create mode 100644 apps/assets/templates/assets/system_user_create.html create mode 100644 apps/assets/templates/assets/system_user_update.html create mode 100644 apps/ops/models.py delete mode 100644 apps/ops/models/__init__.py delete mode 100644 apps/ops/models/ansible.py delete mode 100644 apps/ops/models/task.py delete mode 100644 apps/ops/models/utils.py create mode 100644 apps/ops/tasks.py delete mode 100644 apps/ops/tasks/__init__.py delete mode 100644 apps/ops/tasks/_celery_tasks.py delete mode 100644 apps/ops/tasks/taskers.py diff --git a/apps/assets/forms.py b/apps/assets/forms.py index ec95f0e1d..e6beb3e6e 100644 --- a/apps/assets/forms.py +++ b/apps/assets/forms.py @@ -3,7 +3,10 @@ from django import forms from django.utils.translation import gettext_lazy as _ from .models import IDC, Asset, AssetGroup, AdminUser, SystemUser, Tag -from common.utils import validate_ssh_private_key, ssh_pubkey_gen +from common.utils import validate_ssh_private_key, ssh_pubkey_gen, ssh_key_gen, get_logger + + +logger = get_logger(__file__) class AssetCreateForm(forms.ModelForm): @@ -207,59 +210,59 @@ class SystemUserForm(forms.ModelForm): # Admin user assets define, let user select, save it in form not in view auto_generate_key = forms.BooleanField(initial=True, required=False) # Form field name can not start with `_`, so redefine it, - password = forms.CharField(widget=forms.PasswordInput, max_length=100, min_length=8, strip=True) + password = forms.CharField(widget=forms.PasswordInput, required=False, + max_length=100, min_length=8, strip=True) # Need use upload private key file except paste private key content private_key_file = forms.FileField(required=False) def __init__(self, *args, **kwargs): - # When update a admin user instance, initial it - if kwargs.get('instance'): - initial = kwargs.get('initial', {}) - initial['assets'] = kwargs['instance'].assets.all() - initial['asset_groups'] = kwargs['instance'].asset_groups.all() super(SystemUserForm, self).__init__(*args, **kwargs) def save(self, commit=True): # Because we define custom field, so we need rewrite :method: `save` system_user = super(SystemUserForm, self).save(commit=commit) password = self.cleaned_data['password'] - private_key_file = self.cleaned_data['private_key_file'] + private_key_file = self.cleaned_data.get('private_key_file') if system_user.auth_method == 'P': if password: system_user.password = password - print(password) - # Todo: Validate private key file, and generate public key - # Todo: Auto generate private key and public key - if private_key_file: - system_user.private_key = private_key_file.read().strip() + elif system_user.auth_method == 'K': + if self.cleaned_data['auto_generate_key']: + private_key, public_key = ssh_key_gen(username=system_user.name) + logger.info('Generate private key and public key') + else: + private_key = private_key_file.read().strip() + public_key = ssh_pubkey_gen(private_key=private_key) + system_user.private_key = private_key + system_user.public_key = public_key system_user.save() return self.instance - # Todo: check valid - # def clean_private_key_file(self): - # if not self.cleaned_data['auto_generate_key']: - # if not self.cleaned_data['private_key_file']: - # raise forms.ValidationError(_('Private key required')) - - # def clean_password(self): - # if self.cleaned_data['auth_method'] == 'P': - # if not self.cleaned_data['password']: - # raise forms.ValidationError(_('Password required')) - # return self.cleaned_data['password'] - - # def clean(self): - # password = self.cleaned_data['password'] - # private_key_file = self.cleaned_data.get('private_key_file', '') - # - # if not (password or private_key_file): - # raise forms.ValidationError(_('Password and private key file must be input one')) + def clean_private_key_file(self): + if self.data['auth_method'] == 'K' and \ + not self.cleaned_data['auto_generate_key']: + if not self.cleaned_data['private_key_file']: + raise forms.ValidationError(_('Private key required')) + else: + key_string = self.cleaned_data['private_key_file'].read() + self.cleaned_data['private_key_file'].seek(0) + if not validate_ssh_private_key(key_string): + raise forms.ValidationError(_('Private key invalid')) + return self.cleaned_data['private_key_file'] + + def clean_password(self): + if self.data['auth_method'] == 'P': + if not self.cleaned_data.get('password'): + raise forms.ValidationError(_('Password required')) + return self.cleaned_data['password'] class Meta: model = SystemUser fields = [ - 'name', 'username', 'protocol', 'auto_generate_key', 'password', 'private_key_file', 'auth_method', - 'auto_push', 'sudo', 'comment', 'shell', 'home', 'uid', + 'name', 'username', 'protocol', 'auto_generate_key', 'password', + 'private_key_file', 'auth_method', 'auto_push', 'sudo', + 'comment', 'shell', 'home', 'uid', ] widgets = { 'name': forms.TextInput(attrs={'placeholder': _('Name')}), diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 4f9162de4..4fa70d25e 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -71,7 +71,7 @@ class Asset(models.Model): tags = models.ManyToManyField('Tag', related_name='assets', blank=True, verbose_name=_('Tags')) def __unicode__(self): - return '%(ip)s:%(port)s' % {'ip': self.ip, 'port': self.port} + return '%s <%s: %s>' % (self.hostname, self.ip, self.port) __str__ = __unicode__ diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 94230ad9d..99ad29c6f 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -116,14 +116,13 @@ class SystemUser(models.Model): ) name = models.CharField(max_length=128, unique=True, verbose_name=_('Name')) username = models.CharField(max_length=16, verbose_name=_('Username')) - _password = models.CharField(max_length=256, blank=True, verbose_name=_('Password')) + _password = models.CharField(max_length=256, blank=True, null=True, verbose_name=_('Password')) protocol = models.CharField(max_length=16, choices=PROTOCOL_CHOICES, default='ssh', verbose_name=_('Protocol')) _private_key = models.CharField(max_length=4096, blank=True, verbose_name=_('SSH private key')) _public_key = models.CharField(max_length=4096, blank=True, verbose_name=_('SSH public key')) auth_method = models.CharField(choices=AUTH_METHOD_CHOICES, default='K', max_length=1, verbose_name=_('Auth method')) auto_push = models.BooleanField(default=True, verbose_name=_('Auto push')) - auto_update = models.BooleanField(default=True, verbose_name=_('Auto update pass/key')) sudo = models.TextField(max_length=4096, default='/user/bin/whoami', verbose_name=_('Sudo')) shell = models.CharField(max_length=64, default='/bin/bash', verbose_name=_('Shell')) home = models.CharField(max_length=64, blank=True, verbose_name=_('Home')) @@ -137,7 +136,9 @@ class SystemUser(models.Model): @property def password(self): - return signer.unsign(self._password) + if self._password: + return signer.unsign(self._password) + return None @password.setter def password(self, password_raw): @@ -145,7 +146,9 @@ class SystemUser(models.Model): @property def private_key(self): - return signer.unsign(self._private_key) + if self._private_key: + return signer.unsign(self._private_key) + return None @private_key.setter def private_key(self, private_key_raw): @@ -174,6 +177,17 @@ class SystemUser(models.Model): assets = set(self.assets.all()) | self.get_assets_inherit_from_asset_groups() return list(assets) + def _to_secret_json(self): + """Push system user use it""" + return { + 'name': self.name, + 'username': self.username, + 'shell': self.shell, + 'sudo': self.sudo, + 'password': self.password, + 'public_key': self.public_key + } + @property def assets_amount(self): return self.assets.count() diff --git a/apps/assets/templates/assets/system_user_create_update.html b/apps/assets/templates/assets/_system_user.html similarity index 87% rename from apps/assets/templates/assets/system_user_create_update.html rename to apps/assets/templates/assets/_system_user.html index f07109f91..f9de18e3c 100644 --- a/apps/assets/templates/assets/system_user_create_update.html +++ b/apps/assets/templates/assets/_system_user.html @@ -34,14 +34,20 @@ {% endif %}
{% csrf_token %} + {% if form.non_field_errors %} +
+ {{ form.non_field_errors }} +
+ {% endif %}

{% trans 'Basic' %}

{{ form.name|bootstrap_horizontal }} {{ form.username|bootstrap_horizontal }} {{ form.protocol|bootstrap_horizontal }}

{% trans 'Auth' %}

{{ form.auth_method|bootstrap_horizontal }} + {% block auth %}
@@ -54,6 +60,7 @@ {{ form.private_key_file|bootstrap_horizontal }}
+ {% endblock %}
@@ -88,16 +95,19 @@ $('.password-auth').removeClass('hidden'); $('.public-key-auth').addClass('hidden'); $('#'+'{{ form.password.id_for_label }}').attr('required', 'required'); + $('#'+'{{ form.password.id_for_label }}').removeAttr('disabled'); } else if ($(auth_method).val() == 'K') { $('.password-auth').addClass('hidden'); $('.public-key-auth').removeClass('hidden'); + $('#'+'{{ form.password.id_for_label }}').removeAttr('required'); + $('#'+'{{ form.password.id_for_label }}').attr('disabled', 'disabled'); if ($(auto_generate_key).prop('checked')){ $('#'+'{{ form.private_key_file.id_for_label }}').closest('.form-group').addClass('hidden'); } else { $('#'+'{{ form.private_key_file.id_for_label }}').closest('.form-group').removeClass('hidden'); -{# $('#'+'{{ form.private_key_file.id_for_label }}').attr('required', 'required');#} + $('#'+'{{ form.private_key_file.id_for_label }}').closest('.form-group input').attr('required', 'required'); } } } @@ -110,14 +120,8 @@ $(auto_generate_key).change(function () { authMethodDisplay(); }); + }) - if ($('#'+'{{ form.protocol.id_for_label }}').val() == 'telnet') { - $('#'+'{{ form.auto_generate_key.id_for_label }}').closest('.form-group').remove(); - $('#'+'{{ form.private_key_file.id_for_label }}').closest('.form-group').remove(); - $('#'+'{{ form.auto_push.id_for_label }}').closest('.form-group').remove(); - $('#'+'{{ form.auto_update.id_for_label }}').closest('.form-group').remove(); - } - }) {% endblock %} \ No newline at end of file diff --git a/apps/assets/templates/assets/system_user_create.html b/apps/assets/templates/assets/system_user_create.html new file mode 100644 index 000000000..7127de993 --- /dev/null +++ b/apps/assets/templates/assets/system_user_create.html @@ -0,0 +1,7 @@ +{% extends 'assets/_system_user.html' %} +{% load i18n %} +{% load static %} + +{% block auth %} + {{ block.super }} +{% endblock %} diff --git a/apps/assets/templates/assets/system_user_update.html b/apps/assets/templates/assets/system_user_update.html new file mode 100644 index 000000000..1d63cfcd3 --- /dev/null +++ b/apps/assets/templates/assets/system_user_update.html @@ -0,0 +1,15 @@ +{% extends 'assets/_system_user.html' %} +{% load i18n %} +{% load static %} +{% load bootstrap %} + +{% block auth %} + +
+
+ {{ form.private_key_file|bootstrap_horizontal }} +
+
+{% endblock %} diff --git a/apps/assets/utils.py b/apps/assets/utils.py index e6269e3b1..4eeb3ed65 100644 --- a/apps/assets/utils.py +++ b/apps/assets/utils.py @@ -4,6 +4,7 @@ from rest_framework import serializers from models import Tag from django.views.generic.edit import CreateView + class CreateAssetTagsMiXin(CreateView): def get_form_kwargs(self): tags_list = self.request.POST.getlist('tags') @@ -30,6 +31,7 @@ class CreateAssetTagsMiXin(CreateView): }) return kwargs + class UpdateAssetTagsMiXin(CreateAssetTagsMiXin): def get_form_kwargs(self): kwargs = super(UpdateAssetTagsMiXin, self).get_form_kwargs() diff --git a/apps/assets/views.py b/apps/assets/views.py index df9a4b1e2..75abc50c7 100644 --- a/apps/assets/views.py +++ b/apps/assets/views.py @@ -9,6 +9,7 @@ from openpyxl import load_workbook from django.utils.translation import ugettext as _ from django.conf import settings from django.db.models import Q +from django.db import transaction from django.db import IntegrityError from django.views.generic import TemplateView, ListView, View from django.views.generic.edit import CreateView, DeleteView, FormView, UpdateView @@ -521,9 +522,13 @@ class SystemUserListView(AdminUserRequiredMixin, TemplateView): class SystemUserCreateView(AdminUserRequiredMixin, SuccessMessageMixin, CreateView): model = SystemUser form_class = forms.SystemUserForm - template_name = 'assets/system_user_create_update.html' + template_name = 'assets/system_user_create.html' success_url = reverse_lazy('assets:system-user-list') + @transaction.atomic + def post(self, request, *args, **kwargs): + return super(SystemUserCreateView, self).post(request, *args, **kwargs) + def get_context_data(self, **kwargs): context = { 'app': _('Assets'), @@ -549,7 +554,7 @@ class SystemUserCreateView(AdminUserRequiredMixin, SuccessMessageMixin, CreateVi class SystemUserUpdateView(AdminUserRequiredMixin, UpdateView): model = SystemUser form_class = forms.SystemUserForm - template_name = 'assets/system_user_create_update.html' + template_name = 'assets/system_user_update.html' def get_context_data(self, **kwargs): context = { diff --git a/apps/common/utils.py b/apps/common/utils.py index cccfe1aec..7c0c6c8d4 100644 --- a/apps/common/utils.py +++ b/apps/common/utils.py @@ -16,6 +16,7 @@ import calendar import threading import paramiko +from passlib.hash import sha512_crypt import sshpubkeys from itsdangerous import TimedJSONWebSignatureSerializer, JSONWebSignatureSerializer, \ BadSignature, SignatureExpired @@ -322,4 +323,11 @@ def make_signature(access_key_secret, date=None): return content_md5(data) +def encrypt_password(password): + from passlib.hash import sha512_crypt + if password: + return sha512_crypt.using(rounds=5000).hash(password) + return None + + signer = Signer() \ No newline at end of file diff --git a/apps/ops/models.py b/apps/ops/models.py new file mode 100644 index 000000000..0bdc65ae6 --- /dev/null +++ b/apps/ops/models.py @@ -0,0 +1,31 @@ +# ~*~ coding: utf-8 ~*~ +from __future__ import unicode_literals, absolute_import + +import logging + +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +__all__ = ["TaskRecord"] + + +logger = logging.getLogger(__name__) + + +class TaskRecord(models.Model): + uuid = models.CharField(max_length=128, verbose_name=_('UUID'), primary_key=True) + name = models.CharField(max_length=128, blank=True, verbose_name=_('Name')) + date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Start time')) + date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('End time')) + is_finished = models.BooleanField(default=False, verbose_name=_('Is finished')) + is_success = models.BooleanField(default=False, verbose_name=_('Is success')) + assets = models.TextField(blank=True, null=True, verbose_name=_('Assets')) + result = models.TextField(blank=True, null=True, verbose_name=_('Task result')) + + def __unicode__(self): + return "%s" % self.uuid + + @property + def total_assets(self): + return self.assets.split(',') + diff --git a/apps/ops/models/__init__.py b/apps/ops/models/__init__.py deleted file mode 100644 index dbd842bab..000000000 --- a/apps/ops/models/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from ansible import * -from utils import * -from task import * - diff --git a/apps/ops/models/ansible.py b/apps/ops/models/ansible.py deleted file mode 100644 index 3cf059644..000000000 --- a/apps/ops/models/ansible.py +++ /dev/null @@ -1,291 +0,0 @@ -# ~*~ coding: utf-8 ~*~ -from __future__ import unicode_literals, absolute_import - -import logging -import json - -from django.db import models -from django.utils.translation import ugettext_lazy as _ - -__all__ = ["TaskRecord", "AnsiblePlay", "AnsibleTask", "AnsibleHostResult"] - - -logger = logging.getLogger(__name__) - - -class TaskRecord(models.Model): - uuid = models.CharField(max_length=128, verbose_name=_('UUID'), primary_key=True) - name = models.CharField(max_length=128, blank=True, verbose_name=_('Name')) - start = models.DateTimeField(auto_now_add=True, verbose_name=_('Start Time')) - end = models.DateTimeField(blank=True, null=True, verbose_name=_('End Time')) - exit_code = models.IntegerField(default=0, verbose_name=_('Exit Code')) - completed = models.BooleanField(default=False, verbose_name=_('Is Completed')) - hosts = models.TextField(blank=True, null=True, verbose_name=_('Hosts')) - - def __unicode__(self): - return "%s" % self.uuid - - @property - def total_hosts(self): - return self.hosts.split(',') - - @classmethod - def generate_fake(cls, count=20): - from random import seed - from uuid import uuid4 - import forgery_py - - seed() - for i in range(count): - tasker = cls(uuid=str(uuid4()), - name=forgery_py.name.full_name(), - ) - try: - tasker.save() - logger.debug('Generate fake tasker: %s' % tasker.name) - except Exception as e: - print('Error: %s, continue...' % e.message) - continue - - -class AnsiblePlay(models.Model): - tasker = models.ForeignKey(TaskRecord, related_name='plays', blank=True, null=True) - uuid = models.CharField(max_length=128, verbose_name=_('UUID'), primary_key=True) - name = models.CharField(max_length=128, verbose_name=_('Name')) - - def __unicode__(self): - return "%s<%s>" % (self.name, self.uuid) - - def to_dict(self): - return {"uuid": self.uuid, "name": self.name} - - @classmethod - def generate_fake(cls, count=20): - from random import seed, choice - from uuid import uuid4 - import forgery_py - - seed() - for i in range(count): - play = cls(uuid=str(uuid4()), - name=forgery_py.name.full_name(), - ) - try: - play.tasker = choice(TaskRecord.objects.all()) - play.save() - logger.debug('Generate fake play: %s' % play.name) - except Exception as e: - print('Error: %s, continue...' % e.message) - continue - - -class AnsibleTask(models.Model): - play = models.ForeignKey(AnsiblePlay, related_name='tasks', blank=True, null=True) - uuid = models.CharField(max_length=128, verbose_name=_('UUID'), primary_key=True) - name = models.CharField(max_length=128, blank=True, verbose_name=_('Name')) - - def __unicode__(self): - return "%s<%s>" % (self.name, self.uuid) - - def to_dict(self): - return {"uuid": self.uuid, "name": self.name} - - def failed(self): - pass - - def success(self): - pass - - @classmethod - def generate_fake(cls, count=20): - from random import seed, choice - from uuid import uuid4 - import forgery_py - - seed() - for i in range(count): - task = cls(uuid=str(uuid4()), - name=forgery_py.name.full_name(), - ) - try: - task.play = choice(AnsiblePlay.objects.all()) - task.save() - logger.debug('Generate fake play: %s' % task.name) - except Exception as e: - print('Error: %s, continue...' % e.message) - continue - - -class AnsibleHostResult(models.Model): - task = models.ForeignKey(AnsibleTask, related_name='host_results', blank=True, null=True) - name = models.CharField(max_length=128, blank=True, verbose_name=_('Name')) - success = models.TextField(blank=True, verbose_name=_('Success')) - skipped = models.TextField(blank=True, verbose_name=_('Skipped')) - failed = models.TextField(blank=True, verbose_name=_('Failed')) - unreachable = models.TextField(blank=True, verbose_name=_('Unreachable')) - no_host = models.TextField(blank=True, verbose_name=_('NoHost')) - - def __unicode__(self): - return "%s %s<%s>" % (self.name, str(self.is_success), self.task.uuid) - - @property - def is_failed(self): - if self.failed or self.unreachable or self.no_host: - return True - return False - - @property - def is_success(self): - return not self.is_failed - - @property - def _success_data(self): - if self.success: - return json.loads(self.success) - elif self.skipped: - return json.loads(self.skipped) - - @property - def _failed_data(self): - if self.failed: - return json.loads(self.failed) - elif self.unreachable: - return json.loads(self.unreachable) - elif self.no_host: - return {"msg": self.no_host} - - @property - def failed_msg(self): - return self._failed_data.get("msg") - - @staticmethod - def __filter_disk(ansible_devices, exclude_devices): - """ - 过滤磁盘设备，丢弃掉不需要的设备 - - :param ansible_devices: 对应的facts字段 - :param exclude_devices: 一个需要被丢弃的设备,匹配规则是startwith, 比如需要丢弃sr0子类的 ['sr'] - :return: 过滤获取的结果 - """ - for start_str in exclude_devices: - for key in ansible_devices.keys(): - if key.startswith(start_str): - ansible_devices.pop(key) - return ansible_devices - - @staticmethod - def __filter_interface(ansible_interfaces, exclude_interface): - """ - 过滤网卡设备，丢弃掉不需要的网卡， 比如lo - - :param ansible_interface: 对应的facts字段 - :param exclude_interface: 一个需要被丢弃的设备,匹配规则是startwith, 比如需要丢弃lo子类的 ['lo'] - :return: 过滤获取的结果 - """ - for interface in ansible_interfaces: - for start_str in exclude_interface: - if interface.startswith(start_str): - i = ansible_interfaces.index(interface) - ansible_interfaces.pop(i) - return ansible_interfaces - - @staticmethod - def __gather_interface(facts, interfaces): - """ - 收集所有interface的具体信息 - - :param facts: ansible faces - :param interfaces: 需要收集的intreface列表 - :return: interface的详情 - """ - result = {} - for key in interfaces: - gather_key = "ansible_" + key - if gather_key in facts.keys(): - result[key] = facts.get(gather_key) - return result - - def __deal_setup(self): - """ - 处理ansible setup模块收集到的数据，提取资产需要的部分 - - :return: {"msg": , "data": }, 注意msg是异常信息, 有msg时 data为None - """ - result = self._success_data - module_name = result['invocation'].get('module_name') if result.get('invocation') else None - if module_name is not None: - if module_name != "setup": - return {"msg": "the property only for ansible setup module result!, can't support other module", "data":None} - else: - data = {} - facts =result.get('ansible_facts') - interfaces = self.__filter_interface(facts.get('ansible_interfaces'), ['lo']) - - cpu_describe = "%s %s" % (facts.get('ansible_processor')[0], facts.get('ansible_processor')[1]) if len(facts.get('ansible_processor')) >= 2 else "" - - data['sn'] = facts.get('ansible_product_serial') - data['env'] = facts.get('ansible_env') - data['os'] = "%s %s(%s)" % (facts.get('ansible_distribution'), - facts.get('ansible_distribution_version'), - facts.get('ansible_distribution_release')) - data['mem'] = facts.get('ansible_memtotal_mb') - data['cpu'] = "%s %d核" % (cpu_describe, facts.get('ansible_processor_count')) - data['disk'] = self.__filter_disk(facts.get('ansible_devices'), ['sr']) - data['interface'] = self.__gather_interface(facts, interfaces) - return {"msg": None, "data": data} - else: - return {"msg": "there result isn't ansible setup module result! can't process this data format", "data": None} - - @property - def deal_setup(self): - try: - return self.__deal_setup() - except Exception as e: - return {"msg": "deal with setup data failed, %s" % e.message, "data": None} - - def __deal_ping(self): - """ - 处理ansible ping模块收集到的数据 - - :return: {"msg": , "data": {"success": }}, 注意msg是异常信息, 有msg时 data为None - """ - result = self._success_data - module_name = result['invocation'].get('module_name') if result.get('invocation') else None - if module_name is not None: - if module_name != "ping": - return {"msg": "the property only for ansible setup module result!, can't support other module", "data":None} - else: - ping = True if result.get('ping') == "pong" else False - - return {"msg": None, "data": {"success": ping}} - else: - return {"msg": "there isn't module_name field! can't process this data format", "data": None} - - @property - def deal_ping(self): - try: - return self.__deal_ping() - except Exception as e: - return {"msg": "deal with ping data failed, %s" % e.message, "data": None} - - @classmethod - def generate_fake(cls, count=20): - from random import seed, choice - import forgery_py - - seed() - for i in range(count): - result = cls(name=forgery_py.name.full_name(), - success=forgery_py.lorem_ipsum.sentence(), - failed=forgery_py.lorem_ipsum.sentence(), - skipped=forgery_py.lorem_ipsum.sentence(), - unreachable=forgery_py.lorem_ipsum.sentence(), - no_host=forgery_py.lorem_ipsum.sentence(), - ) - try: - result.task = choice(AnsibleTask.objects.all()) - result.save() - logger.debug('Generate fake HostResult: %s' % result.name) - except Exception as e: - print('Error: %s, continue...' % e.message) - continue diff --git a/apps/ops/models/task.py b/apps/ops/models/task.py deleted file mode 100644 index 9dd3c5b24..000000000 --- a/apps/ops/models/task.py +++ /dev/null @@ -1,32 +0,0 @@ -# ~*~ coding: utf-8 ~*~ -from __future__ import unicode_literals, absolute_import - -import logging - -from uuid import uuid4 -from assets.models import Asset -from ops.models import TaskRecord - -from django.db import models -from django.utils.translation import ugettext_lazy as _ - -__all__ = ["Task"] - - -class Task(models.Model): - """ - Ansible 的Task - """ - name = models.CharField(max_length=128, verbose_name=_('Task name')) - module_name = models.CharField(max_length=128, verbose_name=_('Task module')) - module_args = models.CharField(max_length=512, blank=True, verbose_name=_("Module args")) - - def __unicode__(self): - return "%s" % self.name - - -class Play(models.Model): - """ - Playbook 模板, 定义好Template后生成 Playbook - """ - diff --git a/apps/ops/models/utils.py b/apps/ops/models/utils.py deleted file mode 100644 index 7d9f13e3d..000000000 --- a/apps/ops/models/utils.py +++ /dev/null @@ -1,11 +0,0 @@ -# ~*~ coding: utf-8 ~*~ -from __future__ import unicode_literals - -from ansible import * - -__all__ = ["generate_fake"] - - -def generate_fake(): - for cls in (TaskRecord, AnsiblePlay, AnsibleTask, AnsibleHostResult): - cls.generate_fake() \ No newline at end of file diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py new file mode 100644 index 000000000..45de3e727 --- /dev/null +++ b/apps/ops/tasks.py @@ -0,0 +1,87 @@ +# coding: utf-8 + +from __future__ import absolute_import, unicode_literals +import time + + +from django.utils import timezone +from celery import shared_task + +from common.utils import get_logger, encrypt_password +from .utils.runner import AdHocRunner +from .models import TaskRecord + +logger = get_logger(__file__) + + +@shared_task(name="get_assets_hardware_info") +def get_assets_hardware_info(self, assets): + task_tuple = ( + ('setup', ''), + ) + hoc = AdHocRunner(assets) + return hoc.run(task_tuple) + + +@shared_task(name="asset_test_ping_check") +def asset_test_ping_check(assets): + task_tuple = ( + ('ping', ''), + ) + hoc = AdHocRunner(assets) + result = hoc.run(task_tuple) + return result['contacted'].keys(), result['dark'].keys() + + +@shared_task(bind=True) +def push_users(self, assets, users): + """ + user: { + username: xxx, + shell: /bin/bash, + password: 'staf', + public_key: 'string', + sudo: '/bin/whoami,/sbin/ifconfig' + } + """ + if isinstance(users, dict): + users = [users] + if isinstance(assets, dict): + assets = [assets] + task_tuple = [] + for user in users: + logger.debug('Push user: {}'.format(user)) + # 添加用户, 设置公钥, 设置sudo + task_tuple.extend([ + ('user', 'name={} shell={} state=present password={}'.format( + user['username'], user.get('shell', '/bin/bash'), + encrypt_password(user.get('password', None)))), + ('authorized_key', "user={} state=present key='{}'".format( + user['username'], user['public_key'])), + ('lineinfile', + "name=/etc/sudoers state=present regexp='^{0} ALL=(ALL)' " + "line='{0} ALL=(ALL) NOPASSWD: {1}' " + "validate='visudo -cf %s'".format( + user['username'], user.get('sudo', '/bin/whoami') + )) + ]) + record = TaskRecord(name='Push user', + uuid=self.request.id, + date_start=timezone.now(), + assets=','.join(asset['hostname'] for asset in assets)) + record.save() + logger.info('Runner start {0}'.format(timezone.now())) + hoc = AdHocRunner(assets) + _ = hoc.run(task_tuple) + logger.info('Runner complete {0}'.format(timezone.now())) + result_clean = hoc.clean_result() + record.date_finished = timezone.now() + record.is_finished = True + + if len(result_clean['failed']) == 0: + record.is_success = True + else: + record.is_success = False + record.result = result_clean + record.save() + return result_clean diff --git a/apps/ops/tasks/__init__.py b/apps/ops/tasks/__init__.py deleted file mode 100644 index 6fe598964..000000000 --- a/apps/ops/tasks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from taskers import * \ No newline at end of file diff --git a/apps/ops/tasks/_celery_tasks.py b/apps/ops/tasks/_celery_tasks.py deleted file mode 100644 index 9c76fdc84..000000000 --- a/apps/ops/tasks/_celery_tasks.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import absolute_import, unicode_literals - -from celery import shared_task - -from common import celery_app -from ops.utils.ansible_api import Options, ADHocRunner - - -@shared_task(name="get_asset_hardware_info") -def get_asset_hardware_info(task_name, task_uuid, *assets): - conf = Options() - play_source = { - "name": "Get host hardware information", - "hosts": "default", - "gather_facts": "no", - "tasks": [ - dict(action=dict(module='setup')) - ] - } - hoc = ADHocRunner(conf, play_source, *assets) - ext_code, result = hoc.run(task_name, task_uuid) - return ext_code, result - - -@shared_task(name="asset_test_ping_check") -def asset_test_ping_check(task_name, task_uuid, *assets): - conf = Options() - play_source = { - "name": "Test host connection use ping", - "hosts": "default", - "gather_facts": "no", - "tasks": [ - dict(action=dict(module='ping')) - ] - } - hoc = ADHocRunner(conf, play_source, *assets) - ext_code, result = hoc.run(task_name, task_uuid) - return ext_code, result - - -@shared_task(name="add_user_to_assert") -def add_user_to_asset(): - pass - - -@celery_app.task(name='hello-world') -def hello(): - print('hello world!') diff --git a/apps/ops/tasks/taskers.py b/apps/ops/tasks/taskers.py deleted file mode 100644 index 3a724b049..000000000 --- a/apps/ops/tasks/taskers.py +++ /dev/null @@ -1,126 +0,0 @@ -# ~*~ coding: utf-8 ~*~ -from __future__ import unicode_literals - -from ops.tasks import _celery_tasks - -from ops.models import TaskRecord -from uuid import uuid1 -from celery.result import AsyncResult - -__all__ = ["get_result", - "start_get_hardware_info", - "start_ping_test", - "get_hardware_info", - "get_ping_test"] - - -def get_result(task_id): - result = AsyncResult(task_id) - if result.ready(): - return {"Completed": True, "data": result.get()} - else: - return {"Completed": False, "data": None} - - -def __get_result_by_tasker_id(tasker_uuid, deal_method): - tasker = TaskRecord.objects.get(uuid=tasker_uuid) - total = tasker.total_hosts - total_len = len(total) - host_results = [] - - # 存储数据 - for play in tasker.plays.all(): - for t in play.tasks.all(): - task = {'name': t.name, 'uuid': t.uuid, 'percentage': 0, 'completed': {'success': {}, 'failed': {}}} - completed = [] - count = 0 - for h in t.host_results.all(): - completed.append(h.name) - count += 1 - if h.is_success: - result = getattr(h, deal_method) - if result.get('msg') is None: - task['completed']['success'][h.name] = result.get('data') - else: - task['completed']['failed'][h.name] = result.get('msg') - else: - task['completed']['failed'][h.name] = h.failed_msg - - # 计算进度 - task['percentage'] = float(count * 100 / total_len) - task['waited'] = list(set(total) - set(completed)) - - host_results.append(task) - - return host_results - - -def start_get_hardware_info(*assets): - name = "Get host hardware information" - uuid = "tasker-" + uuid1().hex - _celery_tasks.get_asset_hardware_info.delay(name, uuid, *assets) - return uuid - - -def __get_hardware_info(tasker_uuid): - return __get_result_by_tasker_id(tasker_uuid, 'deal_setup') - - -def get_hardware_info(tasker_uuid): - """ - - :param assets: 资产列表 - :return: 返回数据结构样列 - {u'data': [{u'completed': { - u'failed': {u'192.168.232.135': u'Authentication failure.'}, - u'success': {u'192.168.1.119': {u'cpu': u'GenuineIntel Intel Xeon E312xx (Sandy Bridge) 6\u6838', - u'disk': {: }, - u'env': {: }, - u'interface': {: }, - u'mem': 3951, - u'os': u'Ubuntu 16.04(xenial)', - u'sn': u'NA'}}}, - u'name': u'', - u'percentage': 100.0, - u'uuid': u'87cfedfe-ba55-44ff-bc43-e7e73b869ca1', - u'waited': []} - ], - u'msg': None} - """ - try: - return {"msg": None, "data": __get_hardware_info(tasker_uuid)} - except Exception as e: - return {"msg": "query data failed!, %s" % e.message, "data": None} - - -def start_ping_test(*assets): - name = "Test host connection" - uuid = "tasker-" + uuid1().hex - _celery_tasks.asset_test_ping_check.delay(name, uuid, *assets) - return uuid - - -def __get_ping_test(tasker_uuid): - return __get_result_by_tasker_id(tasker_uuid, 'deal_ping') - - -def get_ping_test(tasker_uuid): - """ - - :param assets: 资产列表 - :return: 返回数据结构样列 - {u'data': [{u'completed': { - u'failed': {u'192.168.232.135': u'Authentication failure.'}, - u'success': {u'192.168.1.119': {u'success': True}}}, - u'name': u'', - u'percentage': 100.0, - u'uuid': u'3e6e0d3b-bee0-4383-b19e-bec6ba55d346', - u'waited': []} - ], - u'msg': None} - """ - try: - return {"msg": None, "data": __get_ping_test(tasker_uuid)} - except Exception as e: - return {"msg": "query data failed!, %s" % e.message, "data": None} - diff --git a/apps/ops/urls/view_urls.py b/apps/ops/urls/view_urls.py index 4b19c3f1c..8a0fba4f7 100644 --- a/apps/ops/urls/view_urls.py +++ b/apps/ops/urls/view_urls.py @@ -9,8 +9,5 @@ __all__ = ["urlpatterns"] urlpatterns = [ # TResource Task url - url(r'^task/list$', page_view.TaskListView.as_view(), name='page-task-list'), - url(r'^task/create$', page_view.TaskCreateView.as_view(), name='page-task-create'), - url(r'^task/(?P[0-9]+)/detail$', page_view.TaskDetailView.as_view(), name='page-task-detail'), - url(r'^task/(?P[0-9]+)/update$', page_view.TaskUpdateView.as_view(), name='page-task-update'), + url(r'^task/list$', page_view.TaskListView.as_view(), name='page-task-list'), ] \ No newline at end of file diff --git a/apps/ops/utils/runner.py b/apps/ops/utils/runner.py index 1f84ad434..14ccf0afe 100644 --- a/apps/ops/utils/runner.py +++ b/apps/ops/utils/runner.py @@ -1,7 +1,7 @@ # ~*~ coding: utf-8 ~*~ +from __future__ import unicode_literals import os -import sys from collections import namedtuple, defaultdict from ansible.executor.task_queue_manager import TaskQueueManager @@ -249,12 +249,20 @@ class AdHocRunner(object): self.loader.cleanup_all_tmp_files() def clean_result(self): - result = defaultdict(dict) - for host, msgs in self.results_callback.result_q['contacted'].items(): - result[host]['success'] = len(msgs) + """ + :return: { + "success": ['hostname',], + "failed": [{'hostname': 'msg'}, {}], + } + """ + result = {'success': [], 'failed': []} + for host in self.results_callback.result_q['contacted']: + result['success'].append(host) for host, msgs in self.results_callback.result_q['dark'].items(): - result[host]['failed'] = len(msgs) + msg = '\n'.join(['{}: {}'.format(msg.get('invocation', {}).get('module_name'), + msg.get('msg', '')) for msg in msgs]) + result['failed'].append({host: msg}) return result diff --git a/apps/ops/views.py b/apps/ops/views.py index c897cde79..ebaa2f033 100644 --- a/apps/ops/views.py +++ b/apps/ops/views.py @@ -2,18 +2,15 @@ from __future__ import unicode_literals from django.conf import settings -from django.views.generic.list import ListView, MultipleObjectMixin -from django.views.generic.edit import CreateView, DeleteView, UpdateView -from django.views.generic.detail import DetailView, SingleObjectMixin +from django.views.generic.list import ListView from users.utils import AdminUserRequiredMixin -from ops.utils.mixins import CreateSudoPrivilegesMixin, ListSudoPrivilegesMixin -from .models import Task +from .models import TaskRecord class TaskListView(AdminUserRequiredMixin, ListView): paginate_by = settings.CONFIG.DISPLAY_PER_PAGE - model = Task + model = TaskRecord context_object_name = 'tasks' template_name = 'task/list.html' @@ -25,18 +22,3 @@ class TaskListView(AdminUserRequiredMixin, ListView): kwargs.update(context) return super(TaskListView, self).get_context_data(**kwargs) - -class TaskCreateView(AdminUserRequiredMixin, CreateView): - model = Task - template_name = 'task/create.html' - - -class TaskUpdateView(AdminUserRequiredMixin, UpdateView): - model = Task - template_name = 'task/update.html' - - -class TaskDetailView(DetailView): - model = Task - context_object_name = 'task' - template_name = 'task/detail.html' diff --git a/apps/perms/hands.py b/apps/perms/hands.py index dd8e61090..cfcae6685 100644 --- a/apps/perms/hands.py +++ b/apps/perms/hands.py @@ -7,9 +7,4 @@ from assets.models import Asset, AssetGroup, SystemUser from assets.serializers import AssetGrantedSerializer, AssetGroupSerializer -def push_system_user(assets, system_user): - print('Push system user %s' % system_user.name) - for asset in assets: - print('\tAsset: %s' % asset.ip) - diff --git a/apps/perms/templates/perms/asset_permission_create_update.html b/apps/perms/templates/perms/asset_permission_create_update.html index 8d7c10924..b2fcb1860 100644 --- a/apps/perms/templates/perms/asset_permission_create_update.html +++ b/apps/perms/templates/perms/asset_permission_create_update.html @@ -38,7 +38,7 @@ {{ form.user_groups|bootstrap_horizontal }}

{% trans 'Asset' %}

- {{ form.assets|bootstrap_horizontal }} + {{ form.assets|bootstrap_horizontal|safe }} {{ form.asset_groups|bootstrap_horizontal }} {{ form.system_users |bootstrap_horizontal }}
diff --git a/apps/perms/utils.py b/apps/perms/utils.py index efad6cbce..a54c47f10 100644 --- a/apps/perms/utils.py +++ b/apps/perms/utils.py @@ -2,9 +2,11 @@ from __future__ import absolute_import, unicode_literals -from common.utils import setattr_bulk -from .hands import User, UserGroup, Asset, AssetGroup, SystemUser, \ - push_system_user +from common.utils import setattr_bulk, get_logger +from ops.tasks import push_users +from .hands import User, UserGroup, Asset, AssetGroup, SystemUser + +logger = get_logger(__file__) def get_user_group_granted_asset_groups(user_group): @@ -220,6 +222,19 @@ def get_users_granted_in_asset_group(asset): pass +def push_system_user(assets, system_user): + logger.info('Push system user %s' % system_user.name) + for asset in assets: + logger.info('\tAsset: %s' % asset.ip) + if not assets: + return None + + assets = [asset._to_secret_json() for asset in assets] + system_user = system_user._to_secret_json() + task = push_users.delay(assets, system_user) + return task.id + + def associate_system_users_and_assets(system_users, assets, asset_groups): """关联系统用户和资产, 目的是保存它们的关系, 然后新加入的资产或系统 用户时,推送系统用户到资产 @@ -242,3 +257,5 @@ def associate_system_users_and_assets(system_users, assets, asset_groups): ) system_user.assets.add(*(tuple(assets_all))) push_system_user(assets_need_push, system_user) + + diff --git a/apps/perms/views.py b/apps/perms/views.py index b1aac2f9a..bce33f23a 100644 --- a/apps/perms/views.py +++ b/apps/perms/views.py @@ -1,9 +1,11 @@ # ~*~ coding: utf-8 ~*~ from __future__ import unicode_literals, absolute_import + import functools from django.utils.translation import ugettext as _ +from django.db import transaction from django.conf import settings from django.db.models import Q from django.views.generic import ListView, CreateView, UpdateView @@ -65,6 +67,10 @@ class AssetPermissionCreateView(AdminUserRequiredMixin, template_name = 'perms/asset_permission_create_update.html' success_url = reverse_lazy('perms:asset-permission-list') + @transaction.atomic + def post(self, request, *args, **kwargs): + return super(AssetPermissionCreateView, self).post(request, *args, **kwargs) + def get_context_data(self, **kwargs): context = { 'app': _('Perms'), diff --git a/requirements/requirements.txt b/requirements/requirements.txt index e3700f963..bc82d5b0b 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -19,3 +19,4 @@ itsdangerous==0.24 tornado==4.4.2 eventlet==0.20.1 django-filter==1.0.0 +passlib==1.7.1 diff --git a/run_server.py b/run_server.py index 384db3282..9f11de4cf 100644 --- a/run_server.py +++ b/run_server.py @@ -29,8 +29,9 @@ def start_django(): def start_celery(): os.chdir(apps_dir) os.environ.setdefault('C_FORCE_ROOT', '1') + os.environ.setdefault('PYTHONOPTIMIZE', 1) print('start celery') - subprocess.call('celery -A common worker -P eventlet -s /tmp/celerybeat-schedule -l info ', shell=True) + subprocess.call('celery -A common worker -s /tmp/celerybeat-schedule -l debug', shell=True) def main(): -- GitLab