!89 删除无用的配置项，sqlhelper的实现优化

Merge pull request !89 from gongzhengtang/master

packageship/packageship/libs/configutils/readconfig.py

@@ -7,8 +7,7 @@
 """
 import configparser
 from configparser import NoSectionError
-from configparser import NoOptionError
-from packageship.system_config import SYS_CONFIG_PATH
+from configparser import NoOptionError, Error
 
 
 class ReadConfig():
@@ -19,9 +18,12 @@ class ReadConfig():
         conf.read:Read the system configuration file
     """
 
-    def __init__(self):
+    def __init__(self, conf_path):
         self.conf = configparser.ConfigParser()
-        self.conf.read(SYS_CONFIG_PATH)
+        if conf_path is None:
+            raise Error('The path of the configuration file does not exist:%s' %
+                        conf_path if conf_path else '')
+        self.conf.read(conf_path)
 
     def get_system(self, param):
         """

packageship/packageship/libs/dbutils/sqlalchemy_helper.py

@@ -18,68 +18,40 @@ from packageship.libs.configutils.readconfig import ReadConfig
 from packageship import system_config
 
 
-class DBHelper():
+class BaseHelper():
     """
-    Description: Database connection, operation public class
-    Attributes:
-        user_name: Username
-        password: Password
-        ip_address: Ip address
-        port: Port
-        db_name: Database name
-        db_type: Database type
-        session: Session
+    Description: Base class for data manipulation
     """
-    # The base class inherited by the data model
-    BASE = declarative_base()
-
-    def __init__(self, user_name=None, password=None, ip_address=None,  # pylint: disable=R0913
-                 port=None, db_name=None, db_type=None, **kwargs):
-        """
-        Description: Class instance initialization
-
-        """
-        self.user_name = user_name
-        self._readconfig = ReadConfig()
-        if self.user_name is None:
-            self.user_name = self._readconfig.get_database('user_name')
-
-        self.password = password
-        if self.password is None:
-            self.password = self._readconfig.get_database('password')
 
-        self.ip_address = ip_address
+    def __init__(self):
+        self.readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
+        self.engine = None
 
-        if self.ip_address is None:
-            self.ip_address = self._readconfig.get_database('host')
 
-        self.port = port
-
-        if self.port is None:
-            self.port = self._readconfig.get_database('port')
-
-        self.db_name = db_name
-
-        if self.db_name is None:
-            self.db_name = self._readconfig.get_database('database')
-
-        self.db_type = db_type
-
-        if self.db_type is None:
-            # read the contents of the configuration file
-            _db_type = self._readconfig.get_database('dbtype')
-            if _db_type is None or _db_type == 'mysql':
-                self.db_type = 'mysql+pymysql'
-            else:
-                self.db_type = 'sqlite:///'
-                if 'import_database' not in kwargs.keys():
-                    self._db_file_path()
-                    self.db_name = os.path.join(
-                        self.database_file_path, self.db_name + '.db')
-        self._create_engine()
-        self.session = None
+class MysqlHelper(BaseHelper):
+    """
+    Description: mysql database connection related operations
+    Attributes:
+        user_name: Database connection username
+        password: Database connection password
+        host: Remote server address
+        port: Port
+        database: Operational database name
+        connection_type: Database connection type
+    """
 
-    def _create_engine(self):
+    def __init__(self, user_name=None, password=None, host=None,  # pylint: disable=unused-argument
+                 port=None, database=None, **kwargs):
+        super(MysqlHelper, self).__init__()
+        self.user_name = user_name or self.readconfig.get_database(
+            'user_name')
+        self.password = password or self.readconfig.get_database('password')
+        self.host = host or self.readconfig.get_database('host')
+        self.port = port or self.readconfig.get_database('port')
+        self.database = database or self.readconfig.get_database('database')
+        self.connection_type = 'mysql+pymysql'
+
+    def create_database_engine(self):
         """
         Description: Create a database connection object
         Args:
@@ -89,29 +61,37 @@ class DBHelper():
             DisconnectionError: A disconnect is detected on a raw DB-API connection.
 
         """
-        if self.db_type.startswith('sqlite'):
-            if not self.db_name:
-                raise DbnameNoneException(
-                    'The connected database name is empty')
-            self.engine = create_engine(
-                self.db_type + self.db_name, encoding='utf-8', convert_unicode=True,
-                connect_args={'check_same_thread': False})
+        if not all([self.user_name, self.password, self.host, self.port, self.database]):
+            raise DisconnectionError(
+                'A disconnect is detected on a raw DB-API connection')
+        # create connection object
+        self.engine = create_engine(URL(**{'database': self.database,
+                                           'username': self.user_name,
+                                           'password': self.password,
+                                           'host': self.host,
+                                           'port': self.port,
+                                           'drivername': self.connection_type}),
+                                    encoding='utf-8',
+                                    convert_unicode=True)
+
+
+class SqliteHlper(BaseHelper):
+    """
+    Description: sqlite database connection related operations
+    Attributes:
+        connection_type: Database connection type
+        database: Operational database name
+    """
+
+    def __init__(self, database, **kwargs):
+        super(SqliteHlper, self).__init__()
+        self.connection_type = 'sqlite:///'
+        if 'complete_route_db' in kwargs.keys():
+            self.database = database
         else:
-            if all([self.user_name, self.password, self.ip_address, self.port, self.db_name]):
-                # create connection object
-                self.engine = create_engine(URL(**{'database': self.db_name,
-                                                   'username': self.user_name,
-                                                   'password': self.password,
-                                                   'host': self.ip_address,
-                                                   'port': self.port,
-                                                   'drivername': self.db_type}),
-                                            encoding='utf-8',
-                                            convert_unicode=True)
-            else:
-                raise DisconnectionError(
-                    'A disconnect is detected on a raw DB-API connection')
-
-    def _db_file_path(self):
+            self.database = self._database_file_path(database)
+
+    def _database_file_path(self, database):
         """
         Description: load the path stored in the sqlite database
         Args:
@@ -120,12 +100,72 @@ class DBHelper():
         Raises:
 
         """
-        self.database_file_path = self._readconfig.get_system(
+        _database_folder_path = self.readconfig.get_system(
             'data_base_path')
-        if not self.database_file_path:
-            self.database_file_path = system_config.DATABASE_FOLDER_PATH
-        if not os.path.exists(self.database_file_path):
-            os.makedirs(self.database_file_path)
+        if not _database_folder_path:
+            _database_folder_path = system_config.DATABASE_FOLDER_PATH
+        try:
+            if not os.path.exists(_database_folder_path):
+                os.makedirs(_database_folder_path)
+        except IOError:
+            pass
+        return os.path.join(_database_folder_path, database + '.db')
+
+    def create_database_engine(self):
+        """
+        Description: Create a database connection object
+        Args:
+
+        Returns:
+        Raises:
+            DisconnectionError: A disconnect is detected on a raw DB-API connection
+
+        """
+        if not self.database:
+            raise DbnameNoneException(
+                'The connected database name is empty')
+        self.engine = create_engine(
+            self.connection_type + self.database, encoding='utf-8', convert_unicode=True,
+            connect_args={'check_same_thread': False})
+
+
+class DBHelper(BaseHelper):
+    """
+    Description: Database connection, operation public class
+    Attributes:
+        user_name: Username
+        password: Password
+        host: Remote server address
+        port: Port
+        db_name: Database name
+        connection_type: Database type
+        session: Session
+    """
+    # The base class inherited by the data model
+    BASE = declarative_base()
+
+    def __init__(self, user_name=None, password=None, host=None,  # pylint: disable=R0913
+                 port=None, db_name=None, connection_type=None, **kwargs):
+        """
+        Description: Class instance initialization
+
+        """
+        super(DBHelper, self).__init__()
+        self._database_engine = {
+            'mysql': MysqlHelper,
+            'sqlite': SqliteHlper
+        }
+        if connection_type is None:
+            connection_type = self.readconfig.get_database(
+                'dbtype') or 'sqlite'
+        _database_engine = self._database_engine.get(connection_type)
+        if _database_engine is None:
+            raise DisconnectionError('')
+        _engine = _database_engine(user_name=user_name, password=password,
+                                   host=host, port=port, database=db_name, **kwargs)
+        _engine.create_database_engine()
+        self.engine = _engine.engine
+        self.session = None
 
     def __enter__(self):
         """
@@ -233,7 +273,7 @@ class DBHelper():
         if model is None:
             raise ContentNoneException('solid model must be specified')
 
-        if dicts is None:
+        if not dicts:
             raise ContentNoneException(
                 'The inserted data content cannot be empty')
 

packageship/packageship/libs/log/loghelper.py

@@ -6,10 +6,11 @@ import os
 import pathlib
 import logging
 from logging.handlers import RotatingFileHandler
-from packageship.system_config import LOG_FOLDER_PATH
+from packageship import system_config
 from packageship.libs.configutils.readconfig import ReadConfig
 
-READCONFIG = ReadConfig()
+
+READCONFIG = ReadConfig(system_config.SYS_CONFIG_PATH)
 
 
 def setup_log(config=None):
@@ -25,10 +26,16 @@ def setup_log(config=None):
         logging.basicConfig(level=_level)
     path = READCONFIG.get_config('LOG', 'log_path')
     log_name = READCONFIG.get_config('LOG', 'log_name')
+    backup_count = READCONFIG.get_config('LOG', 'backup_count')
+    if not backup_count or not isinstance(backup_count, int):
+        backup_count = 10
+    max_bytes = READCONFIG.get_config('LOG', 'max_bytes')
+    if not max_bytes or not isinstance(max_bytes, int):
+        max_bytes = 314572800
     if not log_name:
         log_name = 'log_info.log'
     if not path:
-        path = os.path.join(LOG_FOLDER_PATH, log_name)
+        path = os.path.join(system_config.LOG_FOLDER_PATH, log_name)
     else:
         path = os.path.join(path, log_name)
     if not os.path.exists(path):
@@ -38,7 +45,7 @@ def setup_log(config=None):
             pathlib.Path(path).touch()
 
     file_log_handler = RotatingFileHandler(
-        path, maxBytes=1024 * 1024 * 300, backupCount=10)
+        path, maxBytes=max_bytes, backupCount=backup_count)
 
     formatter = logging.Formatter(
         '%(levelname)s %(filename)s:%(lineno)d %(message)s')
@@ -62,11 +69,12 @@ class Log():
         if not log_name:
             log_name = 'log_info.log'
         if path:
-            self.__path = os.path.join(LOG_FOLDER_PATH, path)
+            self.__path = os.path.join(system_config.LOG_FOLDER_PATH, path)
         else:
             self.__path = READCONFIG.get_config('LOG', 'log_path')
             if not self.__path:
-                self.__path = os.path.join(LOG_FOLDER_PATH, log_name)
+                self.__path = os.path.join(
+                    system_config.LOG_FOLDER_PATH, log_name)
             else:
                 self.__path = os.path.join(self.__path, log_name)
 
@@ -80,30 +88,31 @@ class Log():
             self.__level = 'INFO'
         self.__logger = logging.getLogger(self.__name)
         self.__logger.setLevel(self.__level)
+        self.backup_count = READCONFIG.get_config('LOG', 'backup_count')
+        if not self.backup_count or not isinstance(self.backup_count, int):
+            self.backup_count = 10
+        self.max_bytes = READCONFIG.get_config('LOG', 'max_bytes')
+        if not self.max_bytes or not isinstance(self.max_bytes, int):
+            self.max_bytes = 314572800
 
-    def __ini_handler(self):
-        # self.__stream_handler = logging.StreamHandler()
-        self.__file_handler = logging.FileHandler(
-            self.__path, encoding='utf-8')
+    def __init_handler(self):
+        self.__file_handler = RotatingFileHandler(
+            self.__path, maxBytes=self.max_bytes, backupCount=self.backup_count, encoding="utf-8")
 
     def __set_handler(self):
-        # self.__stream_handler.setLevel(level)
         self.__file_handler.setLevel(self.__level)
-        # self.__logger.addHandler(self.__stream_handler)
         self.__logger.addHandler(self.__file_handler)
 
     def __set_formatter(self):
         formatter = logging.Formatter('%(asctime)s-%(name)s-%(filename)s-[line:%(lineno)d]'
                                       '-%(levelname)s-[ log details ]: %(message)s',
                                       datefmt='%a, %d %b %Y %H:%M:%S')
-        # self.__stream_handler.setFormatter(formatter)
         self.__file_handler.setFormatter(formatter)
 
     def close_handler(self):
         """
             Turn off log processing
         """
-        # self.__stream_handler.close()
         self.__file_handler.close()
 
     @property
@@ -111,7 +120,7 @@ class Log():
         """
             Get logs
         """
-        self.__ini_handler()
+        self.__init_handler()
         self.__set_handler()
         self.__set_formatter()
         self.close_handler()

packageship/packageship/package.ini

@@ -3,9 +3,6 @@
 ; Configuration file path for data initialization
 init_conf_path=/etc/pkgship/conf.yaml
 
-; Whether the system is in debug mode
-debug=false
-
 ; Where to store data files when using sqlite database
 ; data_base_path=/var/run/pkgship_dbs
 
@@ -29,29 +26,6 @@ query_ip_addr=127.0.0.1
 ; call the remote service to complete the data request
 remote_host=https://api.openeuler.org/pkgmanage
 
-
-[DATABASE]
-
-; Basic configuration of sqlalchemy to connect to the database
-
-;Username of the database
-user_name=
-
-;connection password
-password=
-
-;host address
-host=
-
-; number for database connection
-port=
-
-;Connected data name
-database=
-
-; dbtype:The type of database is mainly divided into mysql and sqlite
-dbtype=sqlite
-
 [LOG]
 
 ; Custom log storage path
@@ -65,6 +39,13 @@ log_level=INFO
 ; logging name
 log_name=log_info.log
 
+; The number of dynamically created logs 
+; after the log file size reaches the upper limit. The default is 10 dynamically created logs
+backup_count=10
+
+; The size of each log file, in bytes, the default size of a single log file is 300M
+max_bytes=314572800
+
 [UWSGI]
 ; uwsgi log file path
 daemonize=/var/log/uwsgi.log