Merge remote-tracking branch 'ansgar/package-set' into merge

* ansgar/package-set:
  process-new: allow overrides to work for Package-Set
  Make use of Package-Set in determine_new
  queue.Upload: separate code to load .dsc from check_dsc
  utils.py: Add parser for Package-Set field in .dsc
  process-upload: Dinstall::LintianTags may not be set
Signed-off-by: NJoerg Jaspert <joerg@debian.org>

dak/process_new.py

@@ -424,11 +424,18 @@ def do_new(upload, session):
     # Make a copy of distribution we can happily trample on
     changes["suite"] = copy.copy(changes["distribution"])
 
+    # Try to get an included dsc
+    dsc = None
+    (status, _) = upload.load_dsc()
+    if status:
+        dsc = upload.pkg.dsc
+
     # The main NEW processing loop
     done = 0
+    new = {}
     while not done:
         # Find out what's new
-        new, byhand = determine_new(upload.pkg.changes_file, changes, files, session=session)
+        new, byhand = determine_new(upload.pkg.changes_file, changes, files, dsc=dsc, session=session, new=new)
 
         if not new:
             break
@@ -667,6 +674,12 @@ def do_pkg(changes_full_path, session):
     u.logger = Logger
     origchanges = os.path.abspath(u.pkg.changes_file)
 
+    # Try to get an included dsc
+    dsc = None
+    (status, _) = u.load_dsc()
+    if status:
+        dsc = u.pkg.dsc
+
     cnf = Config()
     bcc = "X-DAK: dak process-new"
     if cnf.has_key("Dinstall::Bcc"):
@@ -691,7 +704,7 @@ def do_pkg(changes_full_path, session):
                 if not recheck(u, session):
                     return
 
-                new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, session=session)
+                new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, dsc=dsc, session=session)
                 if byhand:
                     do_byhand(u, session)
                 elif new:

dak/show_new.py

@@ -180,7 +180,7 @@ def do_pkg(changes_file):
             u.check_source_against_db(deb_filename, session)
     u.pkg.changes["suite"] = u.pkg.changes["distribution"]
 
-    new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, 0, session)
+    new, byhand = determine_new(u.pkg.changes_file, u.pkg.changes, files, 0, dsc=u.pkg.dsc, session=session)
 
     outfile = open(os.path.join(cnf["Show-New::HTMLPath"],htmlname),"w")
 

daklib/queue.py

@@ -51,7 +51,7 @@ from holding import Holding
 from urgencylog import UrgencyLog
 from dbconn import *
 from summarystats import SummaryStats
-from utils import parse_changes, check_dsc_files
+from utils import parse_changes, check_dsc_files, build_package_set
 from textutils import fix_maintainer
 from lintian import parse_lintian_output, generate_reject_messages
 from contents import UnpackedSource
@@ -102,7 +102,7 @@ def get_type(f, session):
 
 # Determine what parts in a .changes are NEW
 
-def determine_new(filename, changes, files, warn=1, session = None):
+def determine_new(filename, changes, files, warn=1, session = None, dsc = None, new = {}):
     """
     Determine what parts in a C{changes} file are NEW.
 
@@ -118,19 +118,30 @@ def determine_new(filename, changes, files, warn=1, session = None):
     @type warn: bool
     @param warn: Warn if overrides are added for (old)stable
 
+    @type dsc: Upload.Pkg.dsc dict
+    @param dsc: (optional); Dsc dictionary
+
+    @type new: dict
+    @param new: new packages as returned by a previous call to this function, but override information may have changed
+
     @rtype: dict
     @return: dictionary of NEW components.
 
     """
     # TODO: This should all use the database instead of parsing the changes
     # file again
-    new = {}
     byhand = {}
 
     dbchg = get_dbchange(filename, session)
     if dbchg is None:
         print "Warning: cannot find changes file in database; won't check byhand"
 
+    # Try to get the Package-Set field from an included .dsc file (if possible).
+    if dsc:
+        for package, entry in build_package_set(dsc, session).items():
+            if not new.has_key(package):
+                new[package] = entry
+
     # Build up a list of potentially new things
     for name, f in files.items():
         # Keep a record of byhand elements
@@ -1105,40 +1116,76 @@ class Upload(object):
                 self.rejects.append("source only uploads are not supported.")
 
     ###########################################################################
-    def check_dsc(self, action=True, session=None):
-        """Returns bool indicating whether or not the source changes are valid"""
-        # Ensure there is source to check
-        if not self.pkg.changes["architecture"].has_key("source"):
-            return True
 
-        # Find the .dsc
+    def __dsc_filename(self):
+        """
+        Returns: (Status, Dsc_Filename)
+        where
+          Status: Boolean; True when there was no error, False otherwise
+          Dsc_Filename: String; name of the dsc file if Status is True, reason for the error otherwise
+        """
         dsc_filename = None
-        for f, entry in self.pkg.files.items():
-            if entry["type"] == "dsc":
+
+        # find the dsc
+        for name, entry in self.pkg.files.items():
+            if entry.has_key("type") and entry["type"] == "dsc":
                 if dsc_filename:
-                    self.rejects.append("can not process a .changes file with multiple .dsc's.")
-                    return False
+                    return False, "cannot process a .changes file with multiple .dsc's."
                 else:
-                    dsc_filename = f
+                    dsc_filename = name
 
-        # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
         if not dsc_filename:
-            self.rejects.append("source uploads must contain a dsc file")
-            return False
+            return False, "source uploads must contain a dsc file"
+
+        return True, dsc_filename
+
+    def load_dsc(self, action=True, signing_rules=1):
+        """
+        Find and load the dsc from self.pkg.files into self.dsc
+
+        Returns: (Status, Reason)
+        where
+          Status: Boolean; True when there was no error, False otherwise
+          Reason: String; When Status is False this describes the error
+        """
+
+        # find the dsc
+        (status, dsc_filename) = self.__dsc_filename()
+        if not status:
+            # If status is false, dsc_filename has the reason
+            return False, dsc_filename
 
-        # Parse the .dsc file
         try:
-            self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1, dsc_file=1))
+            self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=signing_rules, dsc_file=1))
         except CantOpenError:
-            # if not -n copy_to_holding() will have done this for us...
             if not action:
-                self.rejects.append("%s: can't read file." % (dsc_filename))
+                return False, "%s: can't read file." % (dsc_filename)
         except ParseChangesError, line:
-            self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
+            return False, "%s: parse error, can't grok: %s." % (dsc_filename, line)
         except InvalidDscError, line:
-            self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
+            return False, "%s: syntax error on line %s." % (dsc_filename, line)
         except ChangesUnicodeError:
-            self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
+            return False, "%s: dsc file not proper utf-8." % (dsc_filename)
+
+        return True, None
+
+    ###########################################################################
+
+    def check_dsc(self, action=True, session=None):
+        """Returns bool indicating whether or not the source changes are valid"""
+        # Ensure there is source to check
+        if not self.pkg.changes["architecture"].has_key("source"):
+            return True
+
+        (status, reason) = self.load_dsc(action=action)
+        if not status:
+            self.rejects.append(reason)
+            return False
+        (status, dsc_filename) = self.__dsc_filename()
+        if not status:
+            # If status is false, dsc_filename has the reason
+            self.rejects.append(dsc_filename)
+            return False
 
         # Build up the file list of files mentioned by the .dsc
         try:
@@ -1495,7 +1542,7 @@ class Upload(object):
 
         # If we do not have a tagfile, don't do anything
         tagfile = cnf.get("Dinstall::LintianTags")
-        if tagfile is None:
+        if not tagfile:
             return
 
         # Parse the yaml file

daklib/utils.py

@@ -39,7 +39,7 @@ import re
 import email as modemail
 import subprocess
 
-from dbconn import DBConn, get_architecture, get_component, get_suite
+from dbconn import DBConn, get_architecture, get_component, get_suite, get_override_type
 from dak_exceptions import *
 from textutils import fix_maintainer
 from regexes import re_html_escaping, html_escaping, re_single_line_field, \
@@ -588,6 +588,46 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
 
 ################################################################################
 
+# see http://bugs.debian.org/619131
+def build_package_set(dsc, session = None):
+    if not dsc.has_key("package-set"):
+        return {}
+
+    packages = {}
+
+    for line in dsc["package-set"].split("\n"):
+        if not line:
+            break
+
+        (name, section, priority) = line.split()
+        (section, component) = extract_component_from_section(section)
+
+        package_type = "deb"
+        if name.find(":") != -1:
+            (package_type, name) = name.split(":", 1)
+        if package_type == "src":
+            package_type = "dsc"
+
+        # Validate type if we have a session
+        if session and get_override_type(package_type, session) is None:
+            # Maybe just warn and ignore? exit(1) might be a bit hard...
+            utils.fubar("invalid type (%s) in Package-Set." % (package_type))
+
+        if section == "":
+            section = "-"
+        if priority == "":
+            priority = "-"
+
+        if package_type == "dsc":
+            priority = "source"
+
+        if not packages.has_key(name) or packages[name]["type"] == "dsc":
+            packages[name] = dict(priority=priority, section=section, type=package_type, component=component, files=[])
+
+    return packages
+
+################################################################################
+
 def send_mail (message, filename=""):
     """sendmail wrapper, takes _either_ a message string or a file as arguments"""