dominate: Add global dominate query and output

This code can run in no-action mode and provide a complete result in
table format.

dak/dominate.py

@@ -27,7 +27,8 @@ from daklib.config import Config
 from daklib import daklog, utils
 import apt_pkg, sys
 
-from sqlalchemy.sql import exists
+from sqlalchemy.sql import exists, text
+from tabulate import tabulate
 
 
 Options = None
@@ -112,6 +113,182 @@ def doDaDoDa(suite, session):
     idList = obsoleteAllAssociations(suite, session)
     deleteAssociations('bin_associations', idList, session)
 
+
+def retrieve_associations(suites, session):
+    return session.execute(text('''
+WITH
+  -- Provide (source, suite) tuple of all source packages to remain
+  remain_source AS (
+    SELECT
+        *
+      FROM (
+        SELECT
+            source.id AS source_id,
+            src_associations.suite AS suite_id,
+            -- generate rank over versions of a source package in one suite
+            -- "1" being the newest
+            dense_rank() OVER (
+              PARTITION BY source.source, src_associations.suite
+              ORDER BY source.version DESC
+            ) AS version_rank
+          FROM
+            source
+            INNER JOIN src_associations ON
+              src_associations.source = source.id
+              AND src_associations.suite = ANY(:suite_ids)
+        ) AS source_ranked
+      WHERE
+        version_rank = 1
+  ),
+  -- Provide (source, arch, suite) tuple of all binary packages to remain
+  remain_binaries AS (
+    SELECT
+        *
+      FROM (
+        SELECT
+            binaries.id,
+            binaries.architecture AS arch_id,
+            bin_associations.suite AS suite_id,
+            source.id AS source_id,
+            architecture.arch_string AS arch,
+            -- arch of newest version
+            first_value(architecture.arch_string) OVER (
+              PARTITION BY binaries.package, bin_associations.suite
+              ORDER BY binaries.version DESC
+              ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+            ) as arch_first,
+            -- generate rank over versions of a source package in one suite
+            -- "1" being the newest
+            -- if newest package is arch-any, we use the rank only over current arch
+            dense_rank() OVER (
+              PARTITION BY binaries.package, binaries.architecture, bin_associations.suite
+              ORDER BY binaries.version DESC
+            ) AS version_rank_any,
+            -- if newest package is arch-all, we use the rank over all arches
+            -- this makes it possible to replace all by any and any by all
+            dense_rank() OVER (
+              PARTITION BY binaries.package, bin_associations.suite
+              ORDER BY binaries.version DESC
+            ) AS version_rank_all
+          FROM
+            binaries
+            INNER JOIN source ON source.id = binaries.source
+            INNER JOIN bin_associations ON
+              bin_associations.bin = binaries.id
+              AND bin_associations.suite = ANY(:suite_ids)
+            INNER JOIN architecture ON architecture.id = binaries.architecture
+        ) AS source_rank
+      WHERE
+        -- we only want to retain the newest of each
+        CASE
+          WHEN arch != 'all' AND arch_first != 'all' THEN version_rank_any = 1
+          ELSE version_rank_all = 1
+        END
+  ),
+  -- Figure out which source we should remove
+  -- A binary forces the corresponding source to remain
+  dominate_source AS (
+    SELECT
+        source.source AS source_package,
+        source.version AS source_version,
+        source.source AS package,
+        source.version,
+        'source'::text AS arch,
+        suite.suite_name AS suite,
+        src_associations.id AS assoc_id
+      FROM
+        source
+        INNER JOIN src_associations ON
+          src_associations.source = source.id
+          AND src_associations.suite = ANY(:suite_ids)
+        INNER join suite ON suite.id = src_associations.suite
+        LEFT JOIN remain_binaries ON
+          remain_binaries.source_id = source.id
+          AND remain_binaries.suite_id = src_associations.suite
+        LEFT JOIN remain_source ON
+          remain_source.source_id = source.id
+          AND remain_source.suite_id = src_associations.suite
+      WHERE
+        remain_binaries.source_id IS NULL
+        AND remain_source.source_id IS NULL
+  ),
+  -- Figure out which arch-any binaries we should remove
+  dominate_binaries AS (
+    SELECT
+        source.source AS source_package,
+        source.version AS source_version,
+        binaries.package AS package,
+        binaries.version,
+        architecture.arch_string AS arch,
+        suite.suite_name AS suite,
+        bin_associations.id AS assoc_id
+      FROM
+        binaries
+        INNER JOIN source ON source.id = binaries.source
+        INNER JOIN bin_associations ON
+          bin_associations.bin = binaries.id
+          AND bin_associations.suite = ANY(:suite_ids)
+        INNER JOIN architecture ON architecture.id = binaries.architecture
+        INNER join suite ON suite.id = bin_associations.suite
+        LEFT JOIN remain_binaries ON
+          remain_binaries.id = binaries.id
+          AND remain_binaries.arch_id = binaries.architecture
+          AND remain_binaries.suite_id = bin_associations.suite
+      WHERE
+        remain_binaries.source_id IS NULL
+        AND binaries.architecture != (SELECT id from architecture WHERE arch_string = 'all')
+  ),
+  -- Figure out which arch-all binaries we should remove
+  -- A arch-any binary forces the related arch-all binaries to remain
+  dominate_binaries_all AS (
+    SELECT
+        source.source AS source_package,
+        source.version AS source_version,
+        binaries.package AS package,
+        binaries.version,
+        architecture.arch_string AS arch,
+        suite.suite_name AS suite,
+        bin_associations.id AS assoc_id
+      FROM
+        binaries
+        INNER JOIN source ON source.id = binaries.source
+        INNER JOIN bin_associations ON
+          bin_associations.bin = binaries.id
+          AND bin_associations.suite = ANY(:suite_ids)
+        INNER JOIN architecture ON architecture.id = binaries.architecture
+        INNER join suite ON suite.id = bin_associations.suite
+        LEFT JOIN remain_binaries ON
+          remain_binaries.id = binaries.id
+          AND remain_binaries.arch_id = binaries.architecture
+          AND remain_binaries.suite_id = bin_associations.suite
+        LEFT JOIN remain_binaries AS remain_binaries_any ON
+          remain_binaries_any.source_id = source.id
+          AND remain_binaries_any.suite_id = bin_associations.suite
+          AND remain_binaries_any.arch_id != (SELECT id from architecture WHERE arch_string = 'all')
+      WHERE
+        remain_binaries.source_id IS NULL
+        AND remain_binaries_any.source_id IS NULL
+        AND binaries.architecture = (SELECT id from architecture WHERE arch_string = 'all')
+  )
+SELECT
+    *
+  FROM
+    dominate_source
+  UNION SELECT
+    *
+  FROM
+    dominate_binaries
+  UNION SELECT
+    *
+  FROM
+    dominate_binaries_all
+  ORDER BY
+    source_package, source_version, package, version, arch, suite
+''').params(
+    suite_ids = [s.suite_id for s in suites],
+))
+
+
 def usage():
     print """Usage: dak dominate [OPTIONS]
 Remove obsolete source and binary associations from suites.
@@ -154,7 +331,13 @@ def main():
         suites_query = suites_query.filter_by(untouchable = False)
     suites = suites_query.all()
 
-    if not Options['No-Action']:
+    assocs = list(retrieve_associations(suites, session))
+
+    if Options['No-Action']:
+        headers = ('source package', 'source version', 'package', 'version', 'arch', 'suite', 'id')
+        print(tabulate(assocs, headers, tablefmt="orgtbl"))
+
+    else:
         for suite in suites:
             doDaDoDa(suite.suite_id, session)
 

debian/control

@@ -13,6 +13,7 @@ Build-Depends: debhelper (>= 9~),
                python-pytest,
                python-rrdtool,
                python-sqlalchemy,
+               python-tabulate,
                python-yaml
 Maintainer: Debian FTP-Masters <ftpmaster@debian.org>
 Uploaders: Mark Hymers <mhy@debian.org>,
@@ -35,6 +36,7 @@ Depends: binutils-multiarch,
          python-pyrss2gen,
          python-rrdtool,
          python-sqlalchemy,
+         python-tabulate,
          python-yaml,
          symlinks,
          ${python:Depends}