!122 修改单包查询时间过长

Merge pull request !122 from jiangpengjuj/master

packageship/packageship/application/apps/package/function/packages.py

@@ -8,6 +8,7 @@ import math
 
 
 from flask import current_app, jsonify
+from sqlalchemy import text
 from sqlalchemy.exc import SQLAlchemyError, DisconnectionError
 
 
@@ -20,7 +21,6 @@ from packageship.application.models.package import PackagesMaintainer
 from packageship.application.models.package import PackagesIssue
 from packageship.application.models.package import SrcRequires
 from packageship.application.models.package import BinPack
-from packageship.application.models.package import BinRequires
 from packageship.application.models.package import BinProvides
 from packageship.libs.exception import Error
 from packageship.application.models.package import Packages
@@ -159,8 +159,8 @@ def sing_pack(srcname, tablename):
             pack_info_dict = SinglePackInfoSchema(
                 many=False).dump(package_info_obj)
             pack_info_dict = parsing_maintainner(srcname, pack_info_dict)
-            issue_count = database_name.session.query(
-                PackagesIssue).filter_by(pkg_name=package_info_obj.name).count()
+            issue_count = database_name.session.query(PackagesIssue).filter_by(
+                pkg_name=package_info_obj.name).count()
             pack_info_dict["issue"] = issue_count
             buildrequired = buildrequired_search(srcname, tablename)
             pack_info_dict["buildrequired"] = buildrequired
@@ -180,7 +180,8 @@ def sing_pack(srcname, tablename):
             ResponseCode.response_json(
                 ResponseCode.DIS_CONNECTION_DB))
 
-def parsing_maintainner(srcname,pack_info_dict):
+
+def parsing_maintainner(srcname, pack_info_dict):
     """
     Single package query maintainer and maintainlevel
     Args:
@@ -190,8 +191,8 @@ def parsing_maintainner(srcname,pack_info_dict):
 
     """
     with DBHelper(db_name="lifecycle") as database_name:
-        maintainer_obj = database_name.session.query(PackagesMaintainer).filter_by(
-            name=srcname).first()
+        maintainer_obj = database_name.session.query(
+            PackagesMaintainer).filter_by(name=srcname).first()
         if maintainer_obj is None:
             pack_info_dict["maintainer"] = None
             pack_info_dict["maintainlevel"] = None
@@ -238,104 +239,147 @@ def buildrequired_search(srcname, tablename):
         return builddep
 
 
-def _sub_pack(srcname, tablename):
+def helper(cls):
+    """
+    Auxiliary function
+    The returned data format is converted,
+    the main function is to convert a dictionary to a list
+
+    Args:
+        cls: Data before conversion
+    Returns:
+        Converted data
+    """
+    for obj in cls:
+        if "provides" in obj:
+            obj["provides"] = list(obj["provides"].values())
+            for values_p in obj["provides"]:
+                if 'requiredby' in values_p:
+                    values_p['requiredby'] = list(
+                        values_p['requiredby'].values())
+        if "requires" in obj:
+            obj["requires"] = list(obj["requires"].values())
+            for values_r in obj["requires"]:
+                if "providedby" in values_r:
+                    values_r['providedby'] = list(
+                        values_r['providedby'].values())
+
+
+def _sub_pack(src_name, table_name):
     """
     One-level installation dependency of the source package
     to generate the binary package
     Args:
         srcname: The name of the source package
         tablename: The name of the table in the database
-
     Returns:
         One-level installation dependency of the source package to
         generate the binary package
     """
-    with DBHelper(db_name=tablename) as data_name:
-        src_pack_obj = data_name.session.query(
-            SrcPack).filter_by(name=srcname).first()
-        if src_pack_obj is None:
-            return []
-      # Sub-packages generated by the source package
-        bin_pack_set = data_name.session.query(
-            BinPack).filter_by(src_name=src_pack_obj.name).all()
-        pack_list = list()
-        for bin_pack_obj in bin_pack_set:
-            bin_pack_dict = dict()
-            bin_pack_dict['id'] = bin_pack_obj.pkgKey
-            bin_pack_dict['name'] = bin_pack_obj.name
-            # Sub-package lookup provided components
-
-            bin_provides_set = data_name.session.query(
-                BinProvides).filter_by(pkgKey=bin_pack_obj.pkgKey).all()
-            provide_list = provide_(tablename, bin_provides_set)
-            bin_pack_dict['provides'] = provide_list
-            bin_require_set = data_name.session.query(
-                BinRequires).filter_by(pkgKey=bin_pack_obj.pkgKey).all()
-            require_list = require_(tablename, bin_require_set)
-            bin_pack_dict['requires'] = require_list
-            pack_list.append(bin_pack_dict)
-        return pack_list
-
-
-def provide_(tablename, bin_provides_set):
-    """
-    pkgKey goes to the bin_pack table to obtain
-    bin_name corresponding to requiredby
-
-    Args:
-        tablename: table name
-        bin_provides_set: Query set provided by the binary package
-
-    Returns:
-        provide_list: Provided list
-    """
-    with DBHelper(db_name=tablename) as data_name:
-        provide_list = []
-        for bin_provides_obj in bin_provides_set:
-            bin_provides_dict = dict()
-            bin_provides_dict['id'] = bin_provides_obj.id
-            bin_provides_dict['name'] = bin_provides_obj.name
-            reqired_set = data_name.session.query(
-                BinRequires).filter_by(name=bin_provides_obj.name).all()
-            required_pkgkey_list = [
-                reqired_obj.pkgKey for reqired_obj in reqired_set]
-            required_bin_set = data_name.session.query(BinPack).filter(
-                BinPack.pkgKey.in_(required_pkgkey_list)).all()
-            requiredby = [
-                required_bin_obj.name for required_bin_obj in required_bin_set]
-            bin_provides_dict['requiredby'] = requiredby
-            provide_list.append(bin_provides_dict)
-        return provide_list
-
-
-def require_(tablename, bin_require_set):
-    """
-    pkgKey goes to the bin_pack table to obtain
-    bin_name corresponding to provideBy
-
-    Args:
-        tablename: table name
-        bin_provides_set: Query set provided by the binary package
-
-    Returns:
-        require_list: require list
-    """
-    with DBHelper(db_name=tablename) as data_name:
-        # Sub-package to find the required components
-        require_list = []
-
-        for bin_require_obj in bin_require_set:
-            bin_require_dict = dict()
-            bin_require_dict['id'] = bin_require_obj.id
-            bin_require_dict['name'] = bin_require_obj.name
-            provide_set = data_name.session.query(
-                BinProvides).filter_by(name=bin_require_obj.name).all()
-            provide_pkg_list = [
-                provide_obj.pkgKey for provide_obj in provide_set]
-            required_bin_set = data_name.session.query(BinPack).filter(
-                BinPack.pkgKey.in_(provide_pkg_list)).all()
-            providedby = [
-                required_bin_obj.name for required_bin_obj in required_bin_set]
-            bin_require_dict['providedby'] = providedby
-            require_list.append(bin_require_dict)
-        return require_list
+    with DBHelper(db_name=table_name) as database:
+        sql_str = """
+            SELECT DISTINCT
+                b2.pkgKey AS sub_id,
+                b2.name AS sub_name,
+                pro.id AS sub_pro_id,
+                pro.name AS sub_pro_name,
+                b1.name AS sub_reqby_name
+            FROM
+                ( select pkgKey,name,src_name from bin_pack where src_name=:src_name) b2
+                left join bin_provides pro on b2.pkgKey=pro.pkgKey
+                LEFT JOIN bin_requires req ON req.name = pro.name
+                LEFT JOIN bin_pack b1 ON req.pkgKey = b1.pkgKey;
+                """
+        res = {}
+        res_pro = database.session.execute(
+            text(sql_str), {"src_name": src_name}).fetchall()
+
+        for pro_obj in res_pro:
+            if pro_obj.sub_name not in res:
+                res[pro_obj.sub_name] = {
+                    "id": pro_obj.sub_id,
+                    "name": pro_obj.sub_name,
+                    "provides": {
+                        pro_obj.sub_pro_name: {
+                            "id": pro_obj.sub_pro_id,
+                            "name": pro_obj.sub_pro_name,
+                            "requiredby": {
+                                pro_obj.sub_reqby_name: pro_obj.sub_reqby_name
+                            } if pro_obj.sub_reqby_name else {}
+                        }
+                    } if pro_obj.sub_pro_name else {}
+                }
+            else:
+                pro_info = res[pro_obj.sub_name]["provides"]
+                if pro_obj.sub_pro_name in pro_info:
+                    pro_info[pro_obj.sub_pro_name]["requiredby"].update(
+                        {pro_obj.sub_reqby_name: pro_obj.sub_reqby_name})
+                else:
+                    pro_info.update(
+                        {
+                            pro_obj.sub_pro_name: {
+                                "id": pro_obj.sub_pro_id,
+                                "name": pro_obj.sub_pro_name,
+                                "requiredby": {
+                                    pro_obj.sub_reqby_name: pro_obj.sub_reqby_name
+                                } if pro_obj.sub_reqby_name else {}
+                            } if pro_obj.sub_pro_name else {}
+                        }
+                    )
+
+        sql_re = """
+            SELECT DISTINCT
+                b2.pkgKey AS sub_id,
+                b2.name AS sub_name,
+                req.id AS sub_req_id,
+                req.name AS sub_req_name,
+                b1.name AS sub_proby_name
+            FROM
+                ( SELECT pkgKey, name, src_name FROM bin_pack WHERE src_name = :src_name ) b2
+                LEFT JOIN bin_requires req ON b2.pkgKey = req.pkgKey
+                LEFT JOIN bin_provides pro ON pro.name = req.name
+                LEFT JOIN bin_pack b1 ON pro.pkgKey = b1.pkgKey;
+                """
+        res_req = database.session.execute(
+            text(sql_re), {"src_name": src_name}).fetchall()
+
+        for req_obj in res_req:
+            sub_pkg_info = res[req_obj.sub_name]
+            # if req_obj.sub_name not in sub_pkg_info:
+
+            if "requires" not in sub_pkg_info:
+                if not req_obj.sub_req_name:
+                    sub_pkg_info['requires'] = {}
+                else:
+                    sub_pkg_info.update(
+                        {
+                            "requires": {
+                                req_obj.sub_req_name: {
+                                    "id": req_obj.sub_req_id,
+                                    "name": req_obj.sub_req_name,
+                                    "providedby": {
+                                        req_obj.sub_proby_name: req_obj.sub_proby_name
+                                    } if req_obj.sub_proby_name else {}
+                                }
+                            }
+                        }
+                    )
+            else:
+                req_info = sub_pkg_info["requires"]
+                if req_obj.sub_req_name in req_info:
+                    req_info[req_obj.sub_req_name]["providedby"].update(
+                        {req_obj.sub_proby_name: req_obj.sub_proby_name})
+                else:
+                    req_info.update(
+                        {
+                            req_obj.sub_req_name: {
+                                "id": req_obj.sub_req_id,
+                                "name": req_obj.sub_req_name,
+                                "providedby": {
+                                    req_obj.sub_proby_name: req_obj.sub_proby_name
+                                } if req_obj.sub_proby_name else {}
+                            }
+                        }
+                    )
+        helper([values for k, values in res.items()])
+        return (list(res.values()))

packageship/packageship/application/apps/package/serialize.py

@@ -20,7 +20,7 @@ def validate_pagenum(pagenum):
     Raises:
         ValidationError: Test failed
     """
-    if pagenum <= 0 or pagenum >= 65535:
+    if pagenum <= 0 or pagenum > 65535:
         raise ValidationError("pagenum is illegal data ")
 
 
@@ -34,7 +34,7 @@ def validate_pagesize(pagesize):
     Raises:
         ValidationError: Test failed
     """
-    if pagesize <= 0 or pagesize >= 65535:
+    if pagesize <= 0 or pagesize > 65535:
         raise ValidationError("pagesize is illegal data ")
 
 

packageship/packageship/pkgship.py

@@ -511,15 +511,9 @@ class AllPackageCommand(PkgshipCommand):
         self.params = [('tablename', 'str', 'name of the database operated', '', 'store'),
                        ('-remote', 'str', 'The address of the remote service',
                         False, 'store_true'),
-                       ('-pkgname', 'str',
+                       ('-packagename', 'str',
                         'Package name that needs fuzzy matching', '', 'store'),
-                       ('-maintainner', 'str', 'Maintainer\'s name', '', 'store'),
-                       ('-maintainlevel', 'str',
-                        'Maintain the level of data', '', 'store'),
-                       ('-page', 'int',
-                        'Need to query the data on the first few pages', 1, 'store'),
-                       ('-pagesize', 'int',
-                        'The size of the data displayed on each page', 10, 'store')
+                       ('-maintainer', 'str', 'Maintainer\'s name', '', 'store')
                        ]
 
     def register(self):
@@ -577,14 +571,14 @@ class AllPackageCommand(PkgshipCommand):
         self._set_read_host(params.remote)
         _url = self.read_host + \
             '/packages?table_name={table_name}&query_pkg_name={pkg_name}&\
-            maintainner={maintainner}&maintainlevel={maintainlevel}&\
+            maintainner={maintainer}&maintainlevel={maintainlevel}&\
             page_num={page}&page_size={pagesize}'.format(
                 table_name=params.tablename,
-                pkg_name=params.pkgname,
-                maintainner=params.maintainner,
-                maintainlevel=params.maintainlevel,
-                page=params.page,
-                pagesize=params.pagesize).replace(' ', '')
+                pkg_name=params.packagename,
+                maintainer=params.maintainer,
+                maintainlevel='',
+                page=1,
+                pagesize=65535).replace(' ', '')
         try:
             response = requests.get(_url)
         except ConnErr as conn_error:
@@ -600,9 +594,6 @@ class AllPackageCommand(PkgshipCommand):
                     print(response.text)
 
                 if getattr(self.table, 'rowcount'):
-                    print('total count : %d' % response_data['total_count'])
-                    print('total page : %d' % response_data['total_page'])
-                    print('current page : %s ' % params.page)
                     print(self.table)
                 else:
                     print('Sorry, no relevant information has been found yet')
@@ -1382,7 +1373,7 @@ class IssueCommand(PkgshipCommand):
         self.parse = PkgshipCommand.subparsers.add_parser(
             'issue', help='Query the issue list of the specified package')
         self.params = [
-            ('-pkg_name', 'str', 'Query source package name', '', 'store'),
+            ('-packagename', 'str', 'Query source package name', '', 'store'),
 
             ('-issue_type', 'str', 'Type of issue', '', 'store'),
             ('-issue_status', 'str', 'the status of the issue', '', 'store'),
@@ -1390,7 +1381,7 @@ class IssueCommand(PkgshipCommand):
             ('-page', 'int',
              'Need to query the data on the first few pages', 1, 'store'),
             ('-pagesize', 'int',
-             'The size of the data displayed on each page', 100, 'store'),
+             'The size of the data displayed on each page', 65535, 'store'),
             ('-remote', 'str', 'The address of the remote service', False, 'store_true')
         ]
         self.table = self.create_table(
@@ -1448,7 +1439,7 @@ class IssueCommand(PkgshipCommand):
             &issue_status={issue_status}&maintainer={maintainer}'\
                 .format(page_num=params.page,
                         page_size=params.pagesize,
-                        pkg_name=params.pkg_name,
+                        pkg_name=params.packagename,
                         issue_type=params.issue_type,
                         issue_status=params.issue_status,
                         maintainer=params.maintainer).replace(' ', '')

packageship/test/common_files/correct_test_result_json/get_single_package.json

@@ -38,7 +38,6 @@
                         "id": 1,
                         "name": "Ac",
                         "providedby": [
-                            "C1",
                             "C1"
                         ]
                     }
@@ -59,7 +58,6 @@
                         "id": 2,
                         "name": "Ac",
                         "providedby": [
-                            "C1",
                             "C1"
                         ]
                     },

packageship/test/test_module/packages_tests/test_packages.py

@@ -172,7 +172,7 @@ class TestPackages(ReadTestBase):
 
         # test wrong page_num
         resp = self.client.get(
-            "/packages?table_name=mainline&page_num=65535&page_size=1")
+            "/packages?table_name=mainline&page_num=65536&page_size=1")
         resp_dict = json.loads(resp.data)
 
         self.assertIn("code", resp_dict, msg="Error in data format return")
@@ -216,7 +216,7 @@ class TestPackages(ReadTestBase):
 
         # test wrong page_size
         resp = self.client.get(
-            "/packages?table_name=mainline&page_num=1&page_size=65535")
+            "/packages?table_name=mainline&page_num=1&page_size=65536")
         resp_dict = json.loads(resp.data)
 
         self.assertIn("code", resp_dict, msg="Error in data format return")

packageship/test/test_module/single_package_tests/test_get_singlepack.py

@@ -134,7 +134,6 @@ class TestGetSinglePack(ReadTestBase):
         self.assertEqual(ResponseCode.CODE_MSG_MAP.get(ResponseCode.SUCCESS),
                          resp_dict.get("msg"),
                          msg="Error in status prompt return")
-
         self.assertIn("data", resp_dict, msg="Error in data format return")
         self.assertTrue(
             compare_two_values(