# coding: utf-8 # OceanBase Deploy. # Copyright (C) 2021 OceanBase # # This file is part of OceanBase Deploy. # # OceanBase Deploy is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # OceanBase Deploy is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with OceanBase Deploy. If not, see . from __future__ import absolute_import, division, print_function import re import os import sys import time import pickle import string import fcntl import requests from glob import glob from enum import Enum from copy import deepcopy from xml.etree import cElementTree try: from ConfigParser import ConfigParser except: from configparser import ConfigParser from _arch import getArchList, getBaseArch from _rpm import Package, PackageInfo from tool import ConfigUtil, FileUtil, var_replace from _manager import Manager _ARCH = getArchList() _RELEASE = None SUP_MAP = { 'ubuntu': (([16], 7), ), 'debian': (([9], 7), ), 'opensuse-leap': (([15], 7), ), 'sles': (([15, 2], 7), ), 'fedora': (([33], 7), ), 'uos': (([20], 8), ), } _SERVER_VARS = { 'basearch': getBaseArch(), } with FileUtil.open('/etc/os-release') as f: for line in f.readlines(): line = line.strip() if not line: continue try: k, v = line.split('=', 1) _SERVER_VARS[k] = v.strip('"').strip("'") except: pass if 'VERSION_ID' in _SERVER_VARS: m = re.match('\d+', _SERVER_VARS['VERSION_ID']) if m: _RELEASE = m.group(0) _SERVER_VARS['releasever'] = _RELEASE class MirrorRepositoryType(Enum): LOCAL = 'local' REMOTE = 'remote' class MirrorRepository(object): MIRROR_TYPE = None def __init__(self, mirror_path, stdio=None): self.stdio = stdio self.mirror_path = mirror_path self.name = os.path.split(mirror_path)[1] self.section_name = self.name self._str = '%s mirror %s' % (self.mirror_type, self.name) def __str__(self): return self._str @property def mirror_type(self): return self.MIRROR_TYPE def get_all_pkg_info(self): return [] def get_best_pkg(self, **pattern): info = self.get_best_pkg_info(**pattern) return self.get_rpm_pkg_by_info(info) if info else None def get_exact_pkg(self, **pattern): info = self.get_exact_pkg_info(**pattern) return self.get_rpm_pkg_by_info(info) if info else None def _pattern_check(self, pkg, **pattern): for key in ['md5', 'name', 'version', 'release', 'arch']: if pattern.get(key) is not None and getattr(pkg, key) != pattern[key]: self.stdio and getattr(self.stdio, 'verbose', print)('pkg %s is %s, but %s is required' % (key, getattr(pkg, key), pattern[key])) return None return pkg def get_rpm_pkg_by_info(self, pkg_info): return None def get_pkgs_info(self, **pattern): return [] def get_best_pkg_info(self, **pattern): return None def get_exact_pkg_info(self, **pattern): return None def get_pkgs_info_with_score(self, **pattern): return [] class RemotePackageInfo(PackageInfo): def __init__(self, elem): self.epoch = None self.location = (None, None) self.checksum = (None,None) # type,value self.openchecksum = (None,None) # type,value self.time = (None, None) super(RemotePackageInfo, self).__init__(None, None, None, None, None) self._parser(elem) @property def md5(self): return self.checksum[1] @md5.setter def md5(self, value): self.checksum = (self.checksum[0], value) def __str__(self): url = self.location[1] if self.location[0]: url = self.location[0] + url return url def _parser(self, elem): tags = self.__dict__.keys() for child in elem: child_name = RemoteMirrorRepository.ns_cleanup(child.tag) if child_name == 'location': relative = child.attrib.get('href') base = child.attrib.get('base') self.location = (base, relative) elif child_name == 'checksum': csum_value = child.text csum_type = child.attrib.get('type') self.checksum = (csum_type,csum_value) elif child_name == 'open-checksum': csum_value = child.text csum_type = child.attrib.get('type') self.openchecksum = (csum_type, csum_value) elif child_name == 'version': self.epoch = child.attrib.get('epoch') self.set_version(child.attrib.get('ver')) self.set_release(child.attrib.get('rel')) elif child_name == 'time': build = child.attrib.get('build') _file = child.attrib.get('file') self.time = (int(_file), int(build)) elif child_name == 'arch': self.arch = child.text elif child_name == 'name': self.name = child.text class RemoteMirrorRepository(MirrorRepository): class RepoData(object): def __init__(self, elem): self.type = None self.type = elem.attrib.get('type') self.location = (None, None) self.checksum = (None,None) # type,value self.openchecksum = (None,None) # type,value self.timestamp = None self.dbversion = None self.size = None self.opensize = None self.deltas = [] self._parser(elem) def _parser(self, elem): for child in elem: child_name = RemoteMirrorRepository.ns_cleanup(child.tag) if child_name == 'location': relative = child.attrib.get('href') base = child.attrib.get('base') self.location = (base, relative) elif child_name == 'checksum': csum_value = child.text csum_type = child.attrib.get('type') self.checksum = (csum_type,csum_value) elif child_name == 'open-checksum': csum_value = child.text csum_type = child.attrib.get('type') self.openchecksum = (csum_type, csum_value) elif child_name == 'timestamp': self.timestamp = child.text elif child_name == 'database_version': self.dbversion = child.text elif child_name == 'size': self.size = child.text elif child_name == 'open-size': self.opensize = child.text elif child_name == 'delta': delta = RemoteMirrorRepository.RepoData(child) delta.type = self.type self.deltas.append(delta) MIRROR_TYPE = MirrorRepositoryType.REMOTE REMOTE_REPOMD_FILE = '/repodata/repomd.xml' REPOMD_FILE = 'repomd.xml' OTHER_DB_FILE = 'other_db.xml' REPO_AGE_FILE = '.rege_age' DB_CACHE_FILE = '.db' PRIMARY_REPOMD_TYPE = 'primary' def __init__(self, mirror_path, meta_data, stdio=None): self.baseurl = None self.repomd_age = 0 self.repo_age = 0 self.priority = 1 self.gpgcheck = False self._db = None self._repomds = None super(RemoteMirrorRepository, self).__init__(mirror_path, stdio=stdio) self.section_name = meta_data['section_name'] self.baseurl = meta_data['baseurl'] self.enabled = meta_data['enabled'] == '1' self.gpgcheck = ConfigUtil.get_value_from_dict(meta_data, 'gpgcheck', 0, int) > 0 self.priority = 100 - ConfigUtil.get_value_from_dict(meta_data, 'priority', 99, int) if os.path.exists(mirror_path): self._load_repo_age() if self.enabled: repo_age = ConfigUtil.get_value_from_dict(meta_data, 'repo_age', 0, int) if repo_age > self.repo_age or int(time.time()) - 86400 > self.repo_age: self.repo_age = repo_age self.update_mirror() @property def db(self): if self._db is not None: return self._db primary_repomd = self._get_repomd_by_type(self.PRIMARY_REPOMD_TYPE) if not primary_repomd: return [] file_path = self._get_repomd_data_file(primary_repomd) if not file_path: return [] self._load_db_cache(file_path) if self._db is None: fp = FileUtil.unzip(file_path, stdio=self.stdio) if not fp: FileUtil.rm(file_path, stdio=self.stdio) return [] self._db = {} try: parser = cElementTree.iterparse(fp) for event, elem in parser: if RemoteMirrorRepository.ns_cleanup(elem.tag) == 'package' and elem.attrib.get('type') == 'rpm': info = RemotePackageInfo(elem) self._db[info.md5] = info self._dump_db_cache() except: FileUtil.rm(file_path, stdio=self.stdio) self.stdio and self.stdio.critical('failed to parse file %s, please retry later.' % file_path) return [] return self._db def _load_db_cache(self, path): try: db_cacahe_path = self.get_db_cache_file(self.mirror_path) repomd_time = os.stat(path)[8] cache_time = os.stat(db_cacahe_path)[8] if cache_time > repomd_time: self.stdio and getattr(self.stdio, 'verbose', print)('load %s' % db_cacahe_path) with open(db_cacahe_path, 'rb') as f: self._db = pickle.load(f) except: pass def _dump_db_cache(self): if self._db: try: db_cacahe_path = self.get_db_cache_file(self.mirror_path) self.stdio and getattr(self.stdio, 'verbose', print)('dump %s' % db_cacahe_path) with open(db_cacahe_path, 'wb') as f: pickle.dump(self._db, f) return True except: self.stdio.exception('') pass return False @staticmethod def ns_cleanup(qn): return qn if qn.find('}') == -1 else qn.split('}')[1] @staticmethod def get_repo_age_file(mirror_path): return os.path.join(mirror_path, RemoteMirrorRepository.REPO_AGE_FILE) @staticmethod def get_repomd_file(mirror_path): return os.path.join(mirror_path, RemoteMirrorRepository.REPOMD_FILE) @staticmethod def get_other_db_file(mirror_path): return os.path.join(mirror_path, RemoteMirrorRepository.OTHER_DB_FILE) @staticmethod def get_db_cache_file(mirror_path): return os.path.join(mirror_path, RemoteMirrorRepository.DB_CACHE_FILE) def _load_repo_age(self): try: with open(self.get_repo_age_file(self.mirror_path), 'r') as f: self.repo_age = int(f.read()) except: pass def _dump_repo_age_data(self): try: with open(self.get_repo_age_file(self.mirror_path), 'w') as f: f.write(str(self.repo_age)) return True except: pass return False def _get_repomd_by_type(self, repomd_type): repodmds = self.get_repomds() for repodmd in repodmds: if repodmd.type == repomd_type: return repodmd def _get_repomd_data_file(self, repomd): file_name = repomd.location[1] repomd_name = file_name.split('-')[-1] file_path = os.path.join(self.mirror_path, file_name) if os.path.exists(file_path): return file_path base_url = repomd.location[0] if repomd.location[0] else self.baseurl url = '%s/%s' % (base_url, repomd.location[1]) if self.download_file(url, file_path, self.stdio): return file_path def update_mirror(self): self.stdio and getattr(self.stdio, 'start_loading')('Update %s' % self.name) self.get_repomds(True) primary_repomd = self._get_repomd_by_type(self.PRIMARY_REPOMD_TYPE) if not primary_repomd: self.stdio and getattr(self.stdio, 'stop_loading')('fail') return False file_path = self._get_repomd_data_file(primary_repomd) if not file_path: self.stdio and getattr(self.stdio, 'stop_loading')('fail') return False self._db = None self.repo_age = int(time.time()) self._dump_repo_age_data() self.stdio and getattr(self.stdio, 'stop_loading')('succeed') return True def get_repomds(self, update=False): path = self.get_repomd_file(self.mirror_path) if update or not os.path.exists(path): url = '%s/%s' % (self.baseurl, self.REMOTE_REPOMD_FILE) self.download_file(url, path, self.stdio) self._repomds = None if self._repomds is None: self._repomds = [] try: parser = cElementTree.iterparse(path) for event, elem in parser: if RemoteMirrorRepository.ns_cleanup(elem.tag) == 'data': repod = RemoteMirrorRepository.RepoData(elem) self._repomds.append(repod) except: pass return self._repomds def get_all_pkg_info(self): return [self.db[key] for key in self.db] def get_rpm_info_by_md5(self, md5, **pattern): if md5 in self.db: return self._pattern_check(self.db[md5], **pattern) for key in self.db: info = self.db[key] if info.md5 == md5: self.stdio and getattr(self.stdio, 'verbose', print)('%s translate info %s' % (md5, info.md5)) return self._pattern_check(info, **pattern) return None def get_rpm_pkg_by_info(self, pkg_info): file_name = pkg_info.location[1] file_path = os.path.join(self.mirror_path, file_name) self.stdio and getattr(self.stdio, 'verbose', print)('get RPM package by %s' % pkg_info) if not os.path.exists(file_path) or os.stat(file_path)[8] < pkg_info.time[1]: base_url = pkg_info.location[0] if pkg_info.location[0] else self.baseurl url = '%s/%s' % (base_url, pkg_info.location[1]) if not self.download_file(url, file_path, self.stdio): return None return Package(file_path) def get_pkgs_info(self, **pattern): matchs = self.get_pkgs_info_with_score(**pattern) if matchs: return [info[0] for info in sorted(matchs, key=lambda x: x[1], reverse=True)] return matchs def get_best_pkg_info(self, **pattern): matchs = self.get_pkgs_info_with_score(**pattern) if matchs: return Package(max(matchs, key=lambda x: x[1])[0].path) return None def get_exact_pkg_info(self, **pattern): if 'md5' in pattern and pattern['md5']: self.stdio and getattr(self.stdio, 'verbose', print)('md5 is %s' % pattern['md5']) return self.get_rpm_info_by_md5(**pattern) self.stdio and getattr(self.stdio, 'verbose', print)('md5 is None') if 'name' not in pattern and not pattern['name']: self.stdio and getattr(self.stdio, 'verbose', print)('name is None') return None name = pattern['name'] self.stdio and getattr(self.stdio, 'verbose', print)('name is %s' % name) arch = getArchList(pattern['arch']) if 'arch' in pattern and pattern['arch'] else _ARCH self.stdio and getattr(self.stdio, 'verbose', print)('arch is %s' % arch) release = pattern['release'] if 'release' in pattern else None self.stdio and getattr(self.stdio, 'verbose', print)('release is %s' % release) version = pattern['version'] if 'version' in pattern else None self.stdio and getattr(self.stdio, 'verbose', print)('version is %s' % version) pkgs = [] for key in self.db: info = self.db[key] if info.name != name: continue if info.arch not in arch: continue if release and info.release != release: continue if version and version != info.version: continue pkgs.append(info) if pkgs: pkgs.sort() return pkgs[-1] else: return None def get_pkgs_info_with_score(self, **pattern): matchs = [] if 'md5' in pattern and pattern['md5']: self.stdio and getattr(self.stdio, 'verbose', print)('md5 is %s' % pattern['md5']) info = None if pattern['md5'] in self.db: info = self._pattern_check(self.db[pattern['md5']], **pattern) return [info, (0xfffffffff, )] if info else matchs self.stdio and getattr(self.stdio, 'verbose', print)('md5 is None') if 'name' not in pattern and not pattern['name']: self.stdio and getattr(self.stdio, 'verbose', print)('name is None') return matchs self.stdio and getattr(self.stdio, 'verbose', print)('name is %s' % pattern['name']) if 'arch' in pattern and pattern['arch']: pattern['arch'] = getArchList(pattern['arch']) else: pattern['arch'] = _ARCH self.stdio and getattr(self.stdio, 'verbose', print)('arch is %s' % pattern['arch']) release = pattern['release'] if 'release' in pattern else None self.stdio and getattr(self.stdio, 'verbose', print)('release is %s' % release) if 'version' in pattern and pattern['version']: pattern['version'] += '.' else: pattern['version'] = None self.stdio and getattr(self.stdio, 'verbose', print)('version is %s' % pattern['version']) for key in self.db: info = self.db[key] if pattern['name'] in info.name: score = self.match_score(info, **pattern) if score[0]: matchs.append([info, score]) return matchs def match_score(self, info, name, arch, version=None, release=None): if info.arch not in arch: return [0, ] info_version = '%s.' % info.version if version and info_version.find(version) != 0: return [0 ,] if release and info.release != release: raise Exception ('break') return [0 ,] c = [len(name) / len(info.name), info] return c @staticmethod def validate_repoid(repoid): """Return the first invalid char found in the repoid, or None.""" allowed_chars = string.ascii_letters + string.digits + '-_.:' for char in repoid: if char not in allowed_chars: return char else: return None @staticmethod def download_file(url, save_path, stdio=None): try: with requests.get(url, stream=True) as fget: file_size = int(fget.headers["Content-Length"]) if stdio: print_bar = True for func in ['start_progressbar', 'update_progressbar', 'finish_progressbar']: if getattr(stdio, func, False) is False: print_bar = False break else: print_bar = False if print_bar: _, fine_name = os.path.split(save_path) units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} for unit in units: num = file_size / units[unit] if num < 1024: break stdio.start_progressbar('Download %s (%.2f %s)' % (fine_name, num, unit), file_size) chunk_size = 512 file_done = 0 with FileUtil.open(save_path, "wb", stdio=stdio) as fw: for chunk in fget.iter_content(chunk_size): fw.write(chunk) file_done = file_done + chunk_size if print_bar and file_done <= file_size: stdio.update_progressbar(file_done) print_bar and stdio.finish_progressbar() return True except: FileUtil.rm(save_path) stdio and getattr(stdio, 'exception', print)('Failed to download %s to %s' % (url, save_path)) return False class LocalMirrorRepository(MirrorRepository): MIRROR_TYPE = MirrorRepositoryType.LOCAL _DB_FILE = '.db' def __init__(self, mirror_path, stdio=None): super(LocalMirrorRepository, self).__init__(mirror_path, stdio=stdio) self.db = {} self.db_path = os.path.join(mirror_path, self._DB_FILE) self.enabled = '-' self._load_db() @property def repo_age(self): return int(time.time()) def _load_db(self): try: if os.path.isfile(self.db_path): with open(self.db_path, 'rb') as f: db = pickle.load(f) for key in db: data = db[key] path = getattr(data, 'path', False) if not path or not os.path.exists(path): continue self.db[key] = data except: self.stdio.exception('') pass def _dump_db(self): # 所有 dump方案都为临时 try: with open(self.db_path, 'wb') as f: pickle.dump(self.db, f) return True except: self.stdio.exception('') pass return False def exist_pkg(self, pkg): return pkg.md5 in self.db def add_pkg(self, pkg): target_path = os.path.join(self.mirror_path, pkg.file_name) try: src_path = pkg.path self.stdio and getattr(self.stdio, 'verbose', print)('RPM hash check') if target_path != src_path: if pkg.md5 in self.db: t_info = self.db[pkg.md5] self.stdio and getattr(self.stdio, 'verbose', print)('copy %s to %s' % (src_path, target_path)) if t_info.path == target_path: del self.db[t_info.md5] FileUtil.copy(src_path, target_path) else: FileUtil.copy(src_path, target_path) try: self.stdio and getattr(self.stdio, 'verbose', print)('remove %s' % t_info.path) os.remove(t_info.path) except: pass else: FileUtil.copy(src_path, target_path) pkg.path = target_path else: self.stdio and getattr(self.stdio, 'error', print)('same file') return None self.db[pkg.md5] = pkg self.stdio and getattr(self.stdio, 'verbose', print)('dump PackageInfo') if self._dump_db(): self.stdio and getattr(self.stdio, 'print', print)('add %s to local mirror', src_path) return pkg except IOError: self.self.stdio and getattr(self.self.stdio, 'exception', print)('') self.stdio and getattr(self.stdio, 'error', print)('Set local mirror failed. %s IO Error' % pkg.file_name) except: self.stdio and getattr(self.stdio, 'exception', print)('') self.stdio and getattr(self.stdio, 'error', print)('Unable to add %s as local mirror' % pkg.file_name) return None def get_all_pkg_info(self): return [self.db[key] for key in self.db] def get_rpm_pkg_by_info(self, pkg_info): self.stdio and getattr(self.stdio, 'verbose', print)('get RPM package by %s' % pkg_info) return Package(pkg_info.path) def get_pkgs_info(self, **pattern): matchs = self.get_pkgs_info_with_score(**pattern) if matchs: return [info[0] for info in sorted(matchs, key=lambda x: x[1], reverse=True)] return matchs def get_best_pkg_info(self, **pattern): matchs = self.get_pkgs_info_with_score(**pattern) if matchs: return Package(max(matchs, key=lambda x: x[1])[0].path) return None def get_exact_pkg_info(self, **pattern): if 'md5' in pattern and pattern['md5']: self.stdio and getattr(self.stdio, 'verbose', print)('md5 is %s' % pattern['md5']) info = None if pattern['md5'] in self.db: info = self._pattern_check(self.db[pattern['md5']], **pattern) return info self.stdio and getattr(self.stdio, 'verbose', print)('md5 is None') if 'name' not in pattern and not pattern['name']: self.stdio and getattr(self.stdio, 'verbose', print)('name is None') return None name = pattern['name'] self.stdio and getattr(self.stdio, 'verbose', print)('name is %s' % name) arch = getArchList(pattern['arch']) if 'arch' in pattern and pattern['arch'] else _ARCH self.stdio and getattr(self.stdio, 'verbose', print)('arch is %s' % arch) release = pattern['release'] if 'release' in pattern else None self.stdio and getattr(self.stdio, 'verbose', print)('release is %s' % release) version = pattern['version'] if 'version' in pattern else None self.stdio and getattr(self.stdio, 'verbose', print)('version is %s' % version) pkgs = [] for key in self.db: info = self.db[key] if info.name != name: continue if info.arch not in arch: continue if release and info.release != release: continue if version and version != info.version: continue pkgs.append(info) if pkgs: pkgs.sort() return pkgs[-1] else: return None def get_best_pkg_info_with_score(self, **pattern): matchs = self.get_pkgs_info_with_score(**pattern) if matchs: return max(matchs, key=lambda x: x[1]) return None def get_pkgs_info_with_score(self, **pattern): matchs = [] if 'md5' in pattern and pattern['md5']: self.stdio and getattr(self.stdio, 'verbose', print)('md5 is %s' % pattern['md5']) info = None if pattern['md5'] in self.db: info = self._pattern_check(self.db[pattern['md5']], **pattern) return [info, (0xfffffffff, )] if info else matchs self.stdio and getattr(self.stdio, 'verbose', print)('md5 is None') if 'name' not in pattern and not pattern['name']: return matchs self.stdio and getattr(self.stdio, 'verbose', print)('name is %s' % pattern['name']) if 'arch' in pattern and pattern['arch']: pattern['arch'] = getArchList(pattern['arch']) else: pattern['arch'] = _ARCH self.stdio and getattr(self.stdio, 'verbose', print)('arch is %s' % pattern['arch']) release = pattern['release'] if 'release' in pattern else None self.stdio and getattr(self.stdio, 'verbose', print)('release is %s' % release) if 'version' in pattern and pattern['version']: pattern['version'] += '.' else: pattern['version'] = None self.stdio and getattr(self.stdio, 'verbose', print)('version is %s' % pattern['version']) for key in self.db: info = self.db[key] if pattern['name'] in info.name: score = self.match_score(info, **pattern) if score[0]: matchs.append([info, score]) return matchs def match_score(self, info, name, arch, version=None, release=None): if info.arch not in arch: return [0, ] info_version = '%s.' % info.version if version and info_version.find(version) != 0: return [0 ,] if release and info.release != release: return [0 ,] c = [len(name) / len(info.name), info] return c def get_info_list(self): return [self.db[key] for key in self.db] class MirrorRepositoryConfig(object): def __init__(self, path, parser, repo_age): self.path = path self.parser = parser self.repo_age = repo_age self.sections = {} def add_section(self, section): self.sections[section.section_name] = section def __eq__(self, o): return self.repo_age == o.repo_age def __le__(self, o): return self.repo_age < o.repo_age def __gt__(self, o): return self.repo_age > o.repo_age class MirrorRepositorySection(object): def __init__(self, section_name, meta_data, remote_path): self.section_name = section_name self.meta_data = meta_data self.remote_path = remote_path def get_mirror(self, server_vars, stdio=None): meta_data = self.meta_data meta_data['name'] = var_replace(meta_data['name'], server_vars) meta_data['baseurl'] = var_replace(meta_data['baseurl'], server_vars) mirror_path = os.path.join(self.remote_path, meta_data['name']) mirror = RemoteMirrorRepository(mirror_path, meta_data, stdio) return mirror @property def is_enabled(self): return self.meta_data.get('enabled', '1') == '1' class MirrorRepositoryManager(Manager): RELATIVE_PATH = 'mirror' def __init__(self, home_path, lock_manager=None, stdio=None): super(MirrorRepositoryManager, self).__init__(home_path, stdio=stdio) self.remote_path = os.path.join(self.path, 'remote') # rpm remote mirror cache self.local_path = os.path.join(self.path, 'local') self.is_init = self.is_init and self._mkdir(self.remote_path) and self._mkdir(self.local_path) self._local_mirror = None self.lock_manager = lock_manager self._cache_path_repo_config = {} self._cache_section_repo_config = {} def _lock(self, read_only=False): if self.lock_manager: if read_only: return self.lock_manager.mirror_and_repo_sh_lock() else: return self.lock_manager.mirror_and_repo_ex_lock() return True @property def local_mirror(self): self._lock() if self._local_mirror is None: self._local_mirror = LocalMirrorRepository(self.local_path, self.stdio) return self._local_mirror def _get_repo_config(self, path): self.stdio and getattr(self.stdio, 'verbose', print)('load repo config: %s' % path) repo_conf = self._cache_path_repo_config.get(path) repo_age = os.stat(path)[8] if not repo_conf or repo_age != repo_conf.repo_age: with FileUtil.open(path, 'r', stdio=self.stdio) as confpp_obj: parser = ConfigParser() parser.readfp(confpp_obj) repo_conf = MirrorRepositoryConfig(path, parser, repo_age) self._cache_path_repo_config[path] = repo_conf return self._cache_path_repo_config[path] def _get_repo_config_by_section(self, section_name): return self._cache_section_repo_config.get(section_name) def _remove_cache(self, section_name): repo_conf = self._cache_section_repo_config[section_name] del self._cache_path_repo_config[repo_conf.path] del self._cache_section_repo_config[section_name] def _scan_repo_configs(self): cached_sections = list(self._cache_section_repo_config.keys()) for path in glob(os.path.join(self.remote_path, '*.repo')): repo_conf = self._get_repo_config(path) for section in repo_conf.parser.sections(): if section in ['main', 'installed']: continue if section in ['local', 'remote']: self.stdio and getattr(self.stdio, 'warn', print)('%s is system keyword.' % section) continue bad = RemoteMirrorRepository.validate_repoid(section) if bad: continue meta_data = {'section_name': section} for attr in repo_conf.parser.options(section): value = repo_conf.parser.get(section, attr) meta_data[attr] = value if 'enabled' not in meta_data: meta_data['enabled'] = '1' if 'baseurl' not in meta_data: continue if 'name' not in meta_data: meta_data['name'] = section if 'repo_age' not in meta_data: meta_data['repo_age'] = repo_conf.repo_age mirror_section = MirrorRepositorySection(section, meta_data, self.remote_path) repo_conf.add_section(mirror_section) self._cache_section_repo_config[section] = repo_conf if section in cached_sections: cached_sections.remove(section) if cached_sections: for miss_section in cached_sections: self._remove_cache(miss_section) def _get_sections(self): self._scan_repo_configs() sections = {} for repo_conf in sorted(self._cache_path_repo_config.values()): sections.update(repo_conf.sections) return sections.values() def _get_section(self, section_name): self._scan_repo_configs() repo_conf = self._get_repo_config_by_section(section_name) if not repo_conf: return None return repo_conf.sections.get(section_name) def get_remote_mirrors(self, is_enabled=True): self._lock() mirrors = [] server_vars = deepcopy(_SERVER_VARS) linux_id = server_vars.get('ID') if linux_id in SUP_MAP: version = [int(vnum) for vnum in server_vars.get('VERSION_ID', '').split('.')] for vid, elvid in SUP_MAP[linux_id]: if version < vid: break server_vars['releasever'] = elvid server_vars['releasever'] = str(elvid) self.stdio and getattr(self.stdio, 'warn', print)('Use centos %s remote mirror repository for %s %s' % ( server_vars['releasever'], linux_id, server_vars.get('VERSION_ID'))) for mirror_section in self._get_sections(): if is_enabled is not None and is_enabled != mirror_section.is_enabled: continue mirrors.append(mirror_section.get_mirror(server_vars, self.stdio)) return mirrors def get_mirrors(self, is_enabled=True): self._lock() mirrors = self.get_remote_mirrors(is_enabled=is_enabled) mirrors.append(self.local_mirror) return mirrors def get_exact_pkg(self, **pattern): only_info = 'only_info' in pattern and pattern['only_info'] mirrors = self.get_mirrors() info = [None, None] for mirror in mirrors: new_one = mirror.get_exact_pkg_info(**pattern) self.stdio.verbose('%s found pkg: %s' % (mirror, new_one)) if new_one and new_one > info[0]: info = [new_one, mirror] return info[0] if info[0] is None or only_info else info[1].get_rpm_pkg_by_info(info[0]) def get_best_pkg(self, **pattern): if 'fuzzy' not in pattern or not pattern['fuzzy']: return self.get_exact_pkg(**pattern) only_info = 'only_info' in pattern and pattern['only_info'] mirrors = self.get_mirrors() best = None source_mirror = None for mirror in mirrors: t_best = mirror.get_best_pkg_info_with_score(**pattern) self.stdio.verbose('%s found pkg: %s' % (mirror, t_best)) if best is None: best = t_best source_mirror = mirror elif t_best[1] > best[1]: best = t_best source_mirror = mirror if best: return best[0] if only_info else source_mirror.get_rpm_pkg_by_info(best[0]) def get_pkgs_info(self, name, **pattern): pkgs = set() mirrors = self.get_mirrors() for mirror in mirrors: for pkg in mirror.get_pkgs_info(name=name, **pattern): if pkg.name != name: break pkgs.add(pkg) return pkgs def add_remote_mirror(self, src): pass def add_local_mirror(self, src, force=False): self._lock() self.stdio and getattr(self.stdio, 'verbose', print)('%s is file or not' % src) if not os.path.isfile(src): self.stdio and getattr(self.stdio, 'error', print)('No such file: %s' % (src)) return None try: self.stdio and getattr(self.stdio, 'verbose', print)('load %s to Package Object' % src) pkg = Package(src) except: self.stdio and getattr(self.stdio, 'exception', print)('') self.stdio and getattr(self.stdio, 'error', print)('failed to extract info from %s' % src) return None if self.local_mirror.exist_pkg(pkg) and not force: if not self.stdio: return None if not getattr(self.stdio, 'confirm', False): return None if not self.stdio.confirm('mirror %s existed. Do you want to overwrite?' % pkg.file_name): return None self.stdio and getattr(self.stdio, 'print', print)('%s' % pkg) return self.local_mirror.add_pkg(pkg) def set_remote_mirror_enabled(self, section_name, enabled=True): self._lock() op = 'Enable' if enabled else 'Disable' enabled_str = '1' if enabled else '0' self.stdio and getattr(self.stdio, 'start_loading')('%s %s' % (op, section_name)) if section_name == 'local': self.stdio and getattr(self.stdio, 'error', print)('Local mirror repository CANNOT BE %sD.' % op.upper()) return False if section_name == 'remote': self._scan_repo_configs() for repo_config in self._cache_path_repo_config.values(): for section_name in repo_config.sections.keys(): repo_config.parser.set(section_name, 'enabled', enabled_str) with FileUtil.open(repo_config.path, 'w', stdio=self.stdio) as confpp_obj: fcntl.flock(confpp_obj, fcntl.LOCK_EX | fcntl.LOCK_NB) repo_config.parser.write(confpp_obj) repo_age = os.stat(repo_config.path)[8] repo_config.repo_age = repo_age for mirror_section in repo_config.sections.values(): mirror_section.meta_data['enabled'] = enabled_str mirror_section.meta_data['repo_age'] = repo_age self.stdio and getattr(self.stdio, 'stop_loading')('succeed') return True else: mirror_section = self._get_section(section_name) if not mirror_section: self.stdio and getattr(self.stdio, 'error', print)('%s not found.' % (section_name)) self.stdio and getattr(self.stdio, 'stop_loading')('fail') return False if mirror_section.is_enabled == enabled: self.stdio and getattr(self.stdio, 'print', print)('%s is already %sd' % (section_name, op.lower())) self.stdio and getattr(self.stdio, 'stop_loading')('succeed') return True repo_config = self._get_repo_config_by_section(section_name) if not repo_config: self.stdio and getattr(self.stdio, 'error', print)('%s not found.' % (section_name)) self.stdio and getattr(self.stdio, 'stop_loading')('fail') return False repo_config.parser.set(section_name, 'enabled', enabled_str) with FileUtil.open(repo_config.path, 'w', stdio=self.stdio) as confpp_obj: fcntl.flock(confpp_obj, fcntl.LOCK_EX | fcntl.LOCK_NB) repo_config.parser.write(confpp_obj) repo_age = os.stat(repo_config.path)[8] repo_config.repo_age = repo_age mirror_section.meta_data['enabled'] = enabled_str mirror_section.meta_data['repo_age'] = repo_age self.stdio and getattr(self.stdio, 'stop_loading')('succeed') return True