From 39252ca8c5eb0b1dfa04fb1afa3e10384be16ca3 Mon Sep 17 00:00:00 2001 From: Bastian Blank Date: Sat, 22 Jul 2017 14:48:45 +0000 Subject: [PATCH] dominate: Add global dominate query and output This code can run in no-action mode and provide a complete result in table format. --- dak/dominate.py | 187 +++++++++++++++++++++++++++++++++++++++++++++++- debian/control | 2 + 2 files changed, 187 insertions(+), 2 deletions(-) diff --git a/dak/dominate.py b/dak/dominate.py index b50d5183..c7341aaa 100755 --- a/dak/dominate.py +++ b/dak/dominate.py @@ -27,7 +27,8 @@ from daklib.config import Config from daklib import daklog, utils import apt_pkg, sys -from sqlalchemy.sql import exists +from sqlalchemy.sql import exists, text +from tabulate import tabulate Options = None @@ -112,6 +113,182 @@ def doDaDoDa(suite, session): idList = obsoleteAllAssociations(suite, session) deleteAssociations('bin_associations', idList, session) + +def retrieve_associations(suites, session): + return session.execute(text(''' +WITH + -- Provide (source, suite) tuple of all source packages to remain + remain_source AS ( + SELECT + * + FROM ( + SELECT + source.id AS source_id, + src_associations.suite AS suite_id, + -- generate rank over versions of a source package in one suite + -- "1" being the newest + dense_rank() OVER ( + PARTITION BY source.source, src_associations.suite + ORDER BY source.version DESC + ) AS version_rank + FROM + source + INNER JOIN src_associations ON + src_associations.source = source.id + AND src_associations.suite = ANY(:suite_ids) + ) AS source_ranked + WHERE + version_rank = 1 + ), + -- Provide (source, arch, suite) tuple of all binary packages to remain + remain_binaries AS ( + SELECT + * + FROM ( + SELECT + binaries.id, + binaries.architecture AS arch_id, + bin_associations.suite AS suite_id, + source.id AS source_id, + architecture.arch_string AS arch, + -- arch of newest version + first_value(architecture.arch_string) OVER ( + PARTITION BY binaries.package, bin_associations.suite + ORDER BY binaries.version DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) as arch_first, + -- generate rank over versions of a source package in one suite + -- "1" being the newest + -- if newest package is arch-any, we use the rank only over current arch + dense_rank() OVER ( + PARTITION BY binaries.package, binaries.architecture, bin_associations.suite + ORDER BY binaries.version DESC + ) AS version_rank_any, + -- if newest package is arch-all, we use the rank over all arches + -- this makes it possible to replace all by any and any by all + dense_rank() OVER ( + PARTITION BY binaries.package, bin_associations.suite + ORDER BY binaries.version DESC + ) AS version_rank_all + FROM + binaries + INNER JOIN source ON source.id = binaries.source + INNER JOIN bin_associations ON + bin_associations.bin = binaries.id + AND bin_associations.suite = ANY(:suite_ids) + INNER JOIN architecture ON architecture.id = binaries.architecture + ) AS source_rank + WHERE + -- we only want to retain the newest of each + CASE + WHEN arch != 'all' AND arch_first != 'all' THEN version_rank_any = 1 + ELSE version_rank_all = 1 + END + ), + -- Figure out which source we should remove + -- A binary forces the corresponding source to remain + dominate_source AS ( + SELECT + source.source AS source_package, + source.version AS source_version, + source.source AS package, + source.version, + 'source'::text AS arch, + suite.suite_name AS suite, + src_associations.id AS assoc_id + FROM + source + INNER JOIN src_associations ON + src_associations.source = source.id + AND src_associations.suite = ANY(:suite_ids) + INNER join suite ON suite.id = src_associations.suite + LEFT JOIN remain_binaries ON + remain_binaries.source_id = source.id + AND remain_binaries.suite_id = src_associations.suite + LEFT JOIN remain_source ON + remain_source.source_id = source.id + AND remain_source.suite_id = src_associations.suite + WHERE + remain_binaries.source_id IS NULL + AND remain_source.source_id IS NULL + ), + -- Figure out which arch-any binaries we should remove + dominate_binaries AS ( + SELECT + source.source AS source_package, + source.version AS source_version, + binaries.package AS package, + binaries.version, + architecture.arch_string AS arch, + suite.suite_name AS suite, + bin_associations.id AS assoc_id + FROM + binaries + INNER JOIN source ON source.id = binaries.source + INNER JOIN bin_associations ON + bin_associations.bin = binaries.id + AND bin_associations.suite = ANY(:suite_ids) + INNER JOIN architecture ON architecture.id = binaries.architecture + INNER join suite ON suite.id = bin_associations.suite + LEFT JOIN remain_binaries ON + remain_binaries.id = binaries.id + AND remain_binaries.arch_id = binaries.architecture + AND remain_binaries.suite_id = bin_associations.suite + WHERE + remain_binaries.source_id IS NULL + AND binaries.architecture != (SELECT id from architecture WHERE arch_string = 'all') + ), + -- Figure out which arch-all binaries we should remove + -- A arch-any binary forces the related arch-all binaries to remain + dominate_binaries_all AS ( + SELECT + source.source AS source_package, + source.version AS source_version, + binaries.package AS package, + binaries.version, + architecture.arch_string AS arch, + suite.suite_name AS suite, + bin_associations.id AS assoc_id + FROM + binaries + INNER JOIN source ON source.id = binaries.source + INNER JOIN bin_associations ON + bin_associations.bin = binaries.id + AND bin_associations.suite = ANY(:suite_ids) + INNER JOIN architecture ON architecture.id = binaries.architecture + INNER join suite ON suite.id = bin_associations.suite + LEFT JOIN remain_binaries ON + remain_binaries.id = binaries.id + AND remain_binaries.arch_id = binaries.architecture + AND remain_binaries.suite_id = bin_associations.suite + LEFT JOIN remain_binaries AS remain_binaries_any ON + remain_binaries_any.source_id = source.id + AND remain_binaries_any.suite_id = bin_associations.suite + AND remain_binaries_any.arch_id != (SELECT id from architecture WHERE arch_string = 'all') + WHERE + remain_binaries.source_id IS NULL + AND remain_binaries_any.source_id IS NULL + AND binaries.architecture = (SELECT id from architecture WHERE arch_string = 'all') + ) +SELECT + * + FROM + dominate_source + UNION SELECT + * + FROM + dominate_binaries + UNION SELECT + * + FROM + dominate_binaries_all + ORDER BY + source_package, source_version, package, version, arch, suite +''').params( + suite_ids = [s.suite_id for s in suites], +)) + + def usage(): print """Usage: dak dominate [OPTIONS] Remove obsolete source and binary associations from suites. @@ -154,7 +331,13 @@ def main(): suites_query = suites_query.filter_by(untouchable = False) suites = suites_query.all() - if not Options['No-Action']: + assocs = list(retrieve_associations(suites, session)) + + if Options['No-Action']: + headers = ('source package', 'source version', 'package', 'version', 'arch', 'suite', 'id') + print(tabulate(assocs, headers, tablefmt="orgtbl")) + + else: for suite in suites: doDaDoDa(suite.suite_id, session) diff --git a/debian/control b/debian/control index 8ac10fa2..9332a223 100644 --- a/debian/control +++ b/debian/control @@ -13,6 +13,7 @@ Build-Depends: debhelper (>= 9~), python-pytest, python-rrdtool, python-sqlalchemy, + python-tabulate, python-yaml Maintainer: Debian FTP-Masters Uploaders: Mark Hymers , @@ -35,6 +36,7 @@ Depends: binutils-multiarch, python-pyrss2gen, python-rrdtool, python-sqlalchemy, + python-tabulate, python-yaml, symlinks, ${python:Depends} -- GitLab