cross suite version checking. saner othercomponents support.

jennifer

@@ -2,7 +2,7 @@
 
 # Checks Debian packages from Incoming
 # Copyright (C) 2000, 2001, 2002  James Troup <james@nocrew.org>
-# $Id: jennifer,v 1.17 2002-05-10 00:24:33 troup Exp $
+# $Id: jennifer,v 1.18 2002-05-14 15:35:22 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -43,7 +43,7 @@ re_is_changes = re.compile (r"(.+?)_(.+?)_(.+?)\.changes$");
 ################################################################################
 
 # Globals
-jennifer_version = "$Revision: 1.17 $";
+jennifer_version = "$Revision: 1.18 $";
 
 Cnf = None;
 Options = None;
@@ -160,9 +160,11 @@ def get_status_output(cmd, status_read, status_write):
             try:
                 os.close(status_write);
                 os.close(status_read);
+                os.close(c2pread);
                 os.close(c2pwrite);
                 os.close(p2cwrite);
                 os.close(errin);
+                os.close(errout);
             except:
                 pass;
             break;
@@ -207,7 +209,7 @@ def check_signature (filename):
             continue;
         args = split[2:];
         if keywords.has_key(keyword) and keyword != "NODATA":
-            internal_error = internal_error + "found duplicate status token ('%s')." % (keyword);
+            internal_error = internal_error + "found duplicate status token ('%s').\n" % (keyword);
             continue;
         else:
             keywords[keyword] = args;
@@ -431,7 +433,7 @@ def check_distributions():
                 (source, dest) = args[1:3];
                 if changes["distribution"].has_key(source):
                     for arch in changes["architecture"].keys():
-                        if not Cnf.has_key("Suite::%s::Architectures::%s" % (source, arch)):
+                        if arch not in Cnf.ValueList("Suite::%s::Architectures" % (source)):
                             reject("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch),"");
                             del changes["distribution"][source];
                             changes["distribution"][dest] = 1;
@@ -519,13 +521,14 @@ def check_files():
 
             # Ensure the architecture of the .deb is one we know about.
             default_suite = Cnf.get("Dinstall::DefaultSuite", "Unstable")
-            if not Cnf.has_key("Suite::%s::Architectures::%s" % (default_suite, control.Find("Architecture", ""))):
-                reject("Unknown architecture '%s'." % (control.Find("Architecture", "")));
+            architecture = control.Find("Architecture", "");
+            if architecture not in Cnf.ValueList("Suite::%s::Architectures" % (default_suite)):
+                reject("Unknown architecture '%s'." % (architecture));
 
             # Ensure the architecture of the .deb is one of the ones
             # listed in the .changes.
-            if not changes["architecture"].has_key(control.Find("Architecture", "")):
-                reject("%s: control file lists arch as `%s', which isn't in changes file." % (file, control.Find("Architecture", "")));
+            if not changes["architecture"].has_key(architecture):
+                reject("%s: control file lists arch as `%s', which isn't in changes file." % (file, architecture));
 
             # Check the section & priority match those given in the .changes (non-fatal)
             if control.Find("Section") != None and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"):
@@ -534,7 +537,7 @@ def check_files():
                 reject("%s control file lists priority as `%s', but changes file has `%s'." % (file, control.Find("Priority", ""), files[file]["priority"]),"Warning: ");
 
             files[file]["package"] = control.Find("Package");
-            files[file]["architecture"] = control.Find("Architecture");
+            files[file]["architecture"] = architecture;
             files[file]["version"] = control.Find("Version");
             files[file]["maintainer"] = control.Find("Maintainer", "");
             if file[-5:] == ".udeb":
@@ -592,6 +595,8 @@ def check_files():
                         files[file]["new"] = 1;
                     elif not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
                         reject("no source found for %s %s (%s)." % (source_package, source_version, file));
+            # Check the version and for file overwrites
+            reject(Katie.check_binary_against_db(file),"");
 
         # Checks for a source package...
         else:
@@ -636,7 +641,8 @@ def check_files():
                 continue
 
             # Ensure the component is valid for the target suite
-            if Cnf.has_key("Suite:%s::Components" % (suite)) and not Cnf.has_key("Suite::%s::Components::%s" % (suite, files[file]["component"])):
+            if Cnf.has_key("Suite:%s::Components" % (suite)) and \
+               files[file]["component"] not in Cnf.ValueList("Suite::%s::Components" % (suite)):
                 reject("unknown component `%s' for suite `%s'." % (files[file]["component"], suite));
                 continue
 
@@ -644,9 +650,6 @@ def check_files():
             if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file):
                 files[file]["new"] = 1;
 
-            if files[file]["type"] == "deb":
-                reject(Katie.check_binaries_against_db(file, suite),"");
-
             # Validate the component
             component = files[file]["component"];
             component_id = db_access.get_component_id(component);
@@ -675,8 +678,18 @@ def check_files():
             files[file]["files id"] = files_id
 
             # Check for packages that have moved from one component to another
-            if files[file]["oldfiles"].has_key(suite) and files[file]["oldfiles"][suite]["name"] != files[file]["component"]:
-                files[file]["othercomponents"] = files[file]["oldfiles"][suite]["name"];
+            q = Katie.projectB.query("""
+SELECT c.name FROM binaries b, bin_associations ba, suite s, location l,
+                   component c, architecture a, files f
+ WHERE b.package = '%s' AND s.suite_name = '%s'
+   AND (a.arch_string = '%s' OR a.arch_string = 'all')
+   AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id
+   AND f.location = l.id AND l.component = c.id AND b.file = f.id"""
+                               % (files[file]["package"], suite,
+                                  files[file]["architecture"]));
+            ql = q.getresult();
+            if ql:
+                files[file]["othercomponents"] = ql[0][0];
 
     # If the .changes file says it has source, it must have source.
     if changes["architecture"].has_key("source"):
@@ -793,7 +806,7 @@ def check_urgency ():
     if changes["architecture"].has_key("source"):
         if not changes.has_key("urgency"):
             changes["urgency"] = Cnf["Urgency::Default"];
-        if not Cnf.has_key("Urgency::Valid::%s" % changes["urgency"]):
+        if changes["urgency"] not in Cnf.ValueList("Urgency::Valid"):
             reject("%s is not a valid urgency; it will be treated as %s by testing." % (changes["urgency"], Cnf["Urgency::Default"]), "Warning: ");
             changes["urgency"] = Cnf["Urgency::Default"];
         changes["urgency"] = string.lower(changes["urgency"]);

katie

@@ -2,7 +2,7 @@
 
 # Installs Debian packages
 # Copyright (C) 2000, 2001, 2002  James Troup <james@nocrew.org>
-# $Id: katie,v 1.80 2002-05-08 11:52:31 troup Exp $
+# $Id: katie,v 1.81 2002-05-14 15:35:22 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -39,7 +39,7 @@ import db_access, katie, logging, utils;
 ###############################################################################
 
 # Globals
-katie_version = "$Revision: 1.80 $";
+katie_version = "$Revision: 1.81 $";
 
 Cnf = None;
 Options = None;
@@ -118,19 +118,20 @@ def check():
                and not Katie.source_exists(source_package, source_version):
                 reject("no source found for %s %s (%s)." % (source_package, source_version, file));
 
+        # Version and file overwrite checks
+        if not installing_to_stable:
+            if files[file]["type"] == "deb":
+                reject(Katie.check_binary_against_db(file));
+            elif files[file]["type"] == "dsc":
+                reject(Katie.check_source_against_db(file));
+                (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
+                reject(reject_msg);
+
+        # Check the package is still in the override tables
         for suite in changes["distribution"].keys():
-            # Check the package is still in the override tables
             if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file):
                 reject("%s is NEW for %s." % (file, suite));
 
-            if not installing_to_stable:
-                if files[file]["type"] == "deb":
-                    reject(Katie.check_binaries_against_db(file, suite));
-                elif files[file]["type"] == "dsc":
-                    reject(Katie.check_source_against_db(file));
-                    (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file);
-                    reject(reject_msg);
-
 ###############################################################################
 
 def init():

katie.py

@@ -2,7 +2,7 @@
 
 # Utility functions for katie
 # Copyright (C) 2001, 2002  James Troup <james@nocrew.org>
-# $Id: katie.py,v 1.19 2002-05-10 00:24:33 troup Exp $
+# $Id: katie.py,v 1.20 2002-05-14 15:35:22 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -152,7 +152,7 @@ class Katie:
             for i in [ "package", "version", "architecture", "type", "size",
                        "md5sum", "component", "location id", "source package",
                        "source version", "maintainer", "dbtype", "files id",
-                       "new", "section", "priority", "oldfiles", "othercomponents",
+                       "new", "section", "priority", "othercomponents",
                        "pool name" ]:
                 if files[file].has_key(i):
                     d_files[file][i] = files[file][i];
@@ -638,7 +638,7 @@ distribution.""";
             result = q.getresult();
 
         # Remember the section and priority so we can check them later if appropriate
-        if result != []:
+        if result:
             files[file]["override section"] = result[0][0];
             files[file]["override priority"] = result[0][1];
 
@@ -654,21 +654,55 @@ distribution.""";
                 self.reject_message = self.reject_message + "\n";
             self.reject_message = self.reject_message + prefix + str;
 
-    def check_binaries_against_db(self, file, suite):
+    ################################################################################
+
+    def cross_suite_version_check(self, query_result, file, new_version):
+        """Ensure versions are newer than existing packages in target
+        suites and that cross-suite version checking rules as
+        set out in the conf file are satisfied."""
+
+        # Check versions for each target suite
+        for target_suite in self.pkg.changes["distribution"].keys():
+            must_be_newer_than = map(lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)));
+            must_be_older_than = map(lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)));
+            # Enforce "must be newer than target suite" even if conffile omits it
+            if target_suite not in must_be_newer_than:
+                must_be_newer_than.append(target_suite);
+            for entry in query_result:
+                existent_version = entry[0];
+                suite = entry[1];
+                if suite in must_be_newer_than and \
+                   apt_pkg.VersionCompare(new_version, existent_version) != 1:
+                    self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
+                if suite in must_be_older_than and \
+                   apt_pkg.VersionCompare(new_version, existent_version) != -1:
+                    self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
+
+    ################################################################################
+
+    def check_binary_against_db(self, file):
         self.reject_message = "";
         files = self.pkg.files;
 
-        # Find any old binary packages
-        q = self.projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name  FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
-                           % (files[file]["package"], suite, files[file]["architecture"]))
-        for oldfile in q.dictresult():
-            files[file]["oldfiles"][suite] = oldfile;
-            # Check versions [NB: per-suite only; no cross-suite checking done (yet)]
-            if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
-                self.reject("%s: old version (%s) >= new version (%s)." % (file, oldfile["version"], files[file]["version"]));
+        # Ensure version is sane
+        q = self.projectB.query("""
+SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su,
+                                     architecture a
+ WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all')
+   AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
+                                % (files[file]["package"],
+                                   files[file]["architecture"]));
+        self.cross_suite_version_check(q.getresult(), file, files[file]["version"]);
+
         # Check for any existing copies of the file
-        q = self.projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"]))
-        if q.getresult() != []:
+        q = self.projectB.query("""
+SELECT b.id FROM binaries b, architecture a
+ WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s'
+   AND a.id = b.architecture"""
+                                % (files[file]["package"],
+                                   files[file]["version"],
+                                   files[file]["architecture"]))
+        if q.getresult():
             self.reject("can not overwrite existing copy of '%s' already in the archive." % (file));
 
         return self.reject_message;
@@ -676,20 +710,15 @@ distribution.""";
     ################################################################################
 
     def check_source_against_db(self, file):
-        """Ensure source is newer than existing source in target suites."""
         self.reject_message = "";
-        changes = self.pkg.changes;
         dsc = self.pkg.dsc;
 
-        package = dsc.get("source");
-        new_version = dsc.get("version");
-        for suite in changes["distribution"].keys():
-            q = self.projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id"
-                               % (package, suite));
-            ql = map(lambda x: x[0], q.getresult());
-            for old_version in ql:
-                if apt_pkg.VersionCompare(new_version, old_version) != 1:
-                    self.reject("%s: Old version `%s' >= new version `%s'." % (file, old_version, new_version));
+        # Ensure version is sane
+        q = self.projectB.query("""
+SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
+ WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")));
+        self.cross_suite_version_check(q.getresult(), file, dsc.get("version"));
+
         return self.reject_message;
 
     ################################################################################
@@ -742,7 +771,7 @@ distribution.""";
                 q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
                 ql = q.getresult();
 
-                if ql != []:
+                if ql:
                     # Unfortunately, we make get more than one
                     # match here if, for example, the package was
                     # in potato but had a -sa upload in woody.  So