.dsc validation stuff

katie

@@ -2,7 +2,7 @@
 
 # Installs Debian packaes
 # Copyright (C) 2000  James Troup <james@nocrew.org>
-# $Id: katie,v 1.24 2001-01-27 03:51:14 troup Exp $
+# $Id: katie,v 1.25 2001-01-28 09:06:44 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -161,7 +161,7 @@ def check_changes(filename):
 
     # Parse the .changes field into a dictionary [FIXME - need to trap errors, pass on to reject_message etc.]
     try:
-        changes = utils.parse_changes(filename)
+        changes = utils.parse_changes(filename, 0)
     except utils.cant_open_exc:
         reject_message = "Rejected: can't read changes file '%s'.\n" % (filename)
         return 0;
@@ -182,9 +182,6 @@ def check_changes(filename):
             reject_message = "Rejected: Missing field `%s' in changes file.\n" % (i)
             return 0    # Avoid <undef> errors during later tests
 
-    # Fix the Maintainer: field to be RFC822 compatible
-    (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
-
     # Override the Distribution: field if appropriate
     if Cnf["Dinstall::Options::Override-Distribution"] != "":
         reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Cnf["Dinstall::Options::Override-Distribution"])
@@ -199,6 +196,17 @@ def check_changes(filename):
         for j in string.split(o):
             changes[i][j] = 1
 
+    # Fix the Maintainer: field to be RFC822 compatible
+    (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
+
+    # Fix the Changed-By: field to be RFC822 compatible; if it exists.
+    (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by",""));
+
+    # For source uploads the Changed-By field wins; otherwise Maintainer wins.
+    if changes["architecture"].has_key("source"):
+        changes["uploader822"] = "To: %s\nCc: %s" % (changes["changedby822"], changes["maintainer822"]);
+         #         changes["uploadername"], changes["uploaderemail"]) = (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]);
+        
     # Ensure all the values in Closes: are numbers
     if changes.has_key("closes"):
         for i in changes["closes"].keys():
@@ -450,13 +458,16 @@ def check_dsc ():
     for file in files.keys():
         if files[file]["type"] == "dsc":
             try:
-                dsc = utils.parse_changes(file)
+                dsc = utils.parse_changes(file, 1)
             except utils.cant_open_exc:
                 reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename)
                 return 0;
             except utils.changes_parse_error_exc, line:
                 reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
                 return 0;
+            except utils.invalid_dsc_format_exc, line:
+                reject_message = reject_message + "Rejected: syntax error in .dsc file '%s', line %s.\n" % (filename, line)
+                return 0;
             try:
                 dsc_files = utils.build_file_list(dsc, 1)
             except utils.no_files_exc:

melanie

@@ -2,7 +2,7 @@
 
 # General purpose archive tool for ftpmaster
 # Copyright (C) 2000  James Troup <james@nocrew.org>
-# $Id: melanie,v 1.2 2001-01-16 21:52:37 troup Exp $
+# $Id: melanie,v 1.3 2001-01-28 09:06:44 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -184,7 +184,7 @@ def main ():
             for i in source_packages.keys():
                 filename = string.join(source_packages[i], '/');
                 try:
-                    dsc = utils.parse_changes(filename);
+                    dsc = utils.parse_changes(filename, 0);
                 except utils.cant_open_exc:
                     sys.stderr.write("W: couldn't open '%s'.\n" % (filename));
                     continue;

shania

@@ -2,7 +2,7 @@
 
 # Clean incoming of old unused files
 # Copyright (C) 2000  James Troup <james@nocrew.org>
-# $Id: shania,v 1.1 2000-12-13 03:18:50 troup Exp $
+# $Id: shania,v 1.2 2001-01-28 09:06:44 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -93,7 +93,7 @@ def main ():
     # Proces all .changes and .dsc files.
     for changes_filename in changes_files:
         try:
-            changes = utils.parse_changes(changes_filename)
+            changes = utils.parse_changes(changes_filename, 0)
         except:
             continue;
         try:
@@ -105,7 +105,7 @@ def main ():
         for file in files.keys():
             if re_isdsc.search(file) != None:
                 try:
-                    dsc = utils.parse_changes(file)
+                    dsc = utils.parse_changes(file, 0)
                 except:
                     continue;
                 try:

tea

@@ -2,7 +2,7 @@
 
 # Sanity check the database
 # Copyright (C) 2000  James Troup <james@nocrew.org>
-# $Id: tea,v 1.3 2001-01-16 21:52:37 troup Exp $
+# $Id: tea,v 1.4 2001-01-28 09:06:44 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -51,24 +51,9 @@ def process_dir (arg, dirname, filenames):
         if os.path.isfile(filename) and not os.path.islink(filename) and not db_files.has_key(filename) and not excluded.has_key(filename):
             waste = waste + os.stat(filename)[stat.ST_SIZE];
             print filename
-
 ################################################################################
 
-def main ():
-    global Cnf, projectB, db_files, waste, excluded;
-
-    apt_pkg.init();
-    
-    Cnf = apt_pkg.newConfiguration();
-    apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file());
-
-    Arguments = [('d',"debug","Tea::Options::Debug", "IntVal"),
-                 ('h',"help","Tea::Options::Help"),
-                 ('v',"version","Tea::Options::Version")];
-
-    apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
-    projectB = pg.connect('projectb', 'localhost');
-    db_access.init(Cnf, projectB);
+def check_files():
 
     print "Building list of Database files...";
 
@@ -94,6 +79,49 @@ def main ():
     print
     print "%s wasted..." % (utils.size_type(waste));
 
+################################################################################
+
+def check_dscs():
+    count = 0;
+    suite = 'unstable';
+    for component in Cnf.SubTree("Component").List():
+        if component == "mixed":
+            continue;
+        component = string.lower(component);
+        list_filename = '%s%s_%s_source.list' % (Cnf["Dir::ListsDir"], suite, component);
+        list_file = utils.open_file(list_filename, 'r');
+        for line in list_file.readlines():
+            file = line[:-1];
+            try:
+                utils.parse_changes(file, 1);
+            except utils.invalid_dsc_format_exc, line:
+                sys.stderr.write("E: syntax error in .dsc file '%s', line %s.\n" % (file, line));
+                count = count + 1;
+
+    if count:
+        sys.stderr.write("Found %s invalid .dsc files.\n" % (count));
+
+################################################################################
+
+def main ():
+    global Cnf, projectB, db_files, waste, excluded;
+
+    apt_pkg.init();
+    
+    Cnf = apt_pkg.newConfiguration();
+    apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file());
+
+    Arguments = [('d',"debug","Tea::Options::Debug", "IntVal"),
+                 ('h',"help","Tea::Options::Help"),
+                 ('v',"version","Tea::Options::Version")];
+
+    apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
+    projectB = pg.connect('projectb', 'localhost');
+    db_access.init(Cnf, projectB);
+
+    check_dscs();
+    #check_files();
+
 #######################################################################################
 
 if __name__ == '__main__':

test/001/1.dsc

+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.0
+Source: amaya
+Version: 3.2.1-1
+Binary: amaya
+Maintainer: Steve Dunham <dunham@debian.org>
+Architecture: any
+Standards-Version: 2.4.0.0
+Files: 
+ 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz
+ da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.0.2 (GNU/Linux)
+Comment: For info see http://www.gnupg.org
+
+iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ
+rhYnRmVuNMa8oYSvL4hl/Yw=
+=EFAA
+-----END PGP SIGNATURE-----

test/001/2.dsc

+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.0
+Source: amaya
+Version: 3.2.1-1
+Binary: amaya
+Maintainer: Steve Dunham <dunham@debian.org>
+Architecture: any
+Standards-Version: 2.4.0.0
+Files: 
+ 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz
+ da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.0.2 (GNU/Linux)
+Comment: For info see http://www.gnupg.org
+
+iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ
+rhYnRmVuNMa8oYSvL4hl/Yw=
+=EFAA
+-----END PGP SIGNATURE-----

test/001/3.dsc

+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+Format: 1.0
+Source: amaya
+Version: 3.2.1-1
+Binary: amaya
+Maintainer: Steve Dunham <dunham@debian.org>
+Architecture: any
+Standards-Version: 2.4.0.0
+Files: 
+ 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz
+ da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.0.2 (GNU/Linux)
+Comment: For info see http://www.gnupg.org
+
+iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ
+rhYnRmVuNMa8oYSvL4hl/Yw=
+=EFAA
+-----END PGP SIGNATURE-----

test/001/4.dsc

+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+Format: 1.0
+Source: amaya
+Version: 3.2.1-1
+Binary: amaya
+Maintainer: Steve Dunham <dunham@debian.org>
+Architecture: any
+Standards-Version: 2.4.0.0
+Files: 
+ 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz
+ da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.0.2 (GNU/Linux)
+Comment: For info see http://www.gnupg.org
+iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ
+rhYnRmVuNMa8oYSvL4hl/Yw=
+=EFAA
+-----END PGP SIGNATURE-----

test/001/5.dsc

+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.0
+Source: amaya
+Version: 3.2.1-1
+Binary: amaya
+Maintainer: Steve Dunham <dunham@debian.org>
+Architecture: any
+Standards-Version: 2.4.0.0
+Files: 
+ 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz
+ da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz
+
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.0.2 (GNU/Linux)
+Comment: For info see http://www.gnupg.org
+
+iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ
+rhYnRmVuNMa8oYSvL4hl/Yw=
+=EFAA
+-----END PGP SIGNATURE-----

test/001/6.dsc

+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+
+Format: 1.0
+Source: amaya
+Version: 3.2.1-1
+Binary: amaya
+Maintainer: Steve Dunham <dunham@debian.org>
+Architecture: any
+Standards-Version: 2.4.0.0
+Files: 
+ 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz
+ da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.0.2 (GNU/Linux)
+Comment: For info see http://www.gnupg.org
+
+iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ
+rhYnRmVuNMa8oYSvL4hl/Yw=
+=EFAA
+-----END PGP SIGNATURE-----

test/001/test.py

+#!/usr/bin/env python
+
+# Check utils.parse_changes()'s .dsc file validation
+# Copyright (C) 2000  James Troup <james@nocrew.org>
+# $Id: test.py,v 1.1 2001-01-28 09:06:44 troup Exp $
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+################################################################################
+
+import os, sys
+
+sys.path.append(os.path.abspath('../../'));
+
+import utils
+
+################################################################################
+
+def fail(message):
+    sys.stderr.write("%s\n" % (message));
+    sys.exit(1);
+    
+################################################################################
+
+def main ():
+    # Valid .dsc
+    utils.parse_changes('1.dsc',1);
+
+    # Missing blank line before signature body
+    try:
+        utils.parse_changes('2.dsc',1);
+    except utils.invalid_dsc_format_exc, line:
+        if line != 14:
+            fail("Incorrect line number ('%s') for test #2." % (line));
+    else:
+        fail("Test #2 wasn't recognised as invalid.");
+
+    # Missing blank line after signature header
+    try:
+        utils.parse_changes('3.dsc',1);
+    except utils.invalid_dsc_format_exc, line:
+        if line != 14:
+            fail("Incorrect line number ('%s') for test #3." % (line));
+    else:
+        fail("Test #3 wasn't recognised as invalid.");
+
+    # No blank lines at all
+    try:
+        utils.parse_changes('4.dsc',1);
+    except utils.invalid_dsc_format_exc, line:
+        if line != 19:
+            fail("Incorrect line number ('%s') for test #4." % (line));
+    else:
+        fail("Test #4 wasn't recognised as invalid.");
+
+    # Extra blank line before signature body
+    try:
+        utils.parse_changes('5.dsc',1);
+    except utils.invalid_dsc_format_exc, line:
+        if line != 15:
+            fail("Incorrect line number ('%s') for test #5." % (line));
+    else:
+        fail("Test #5 wasn't recognised as invalid.");
+
+    # Extra blank line after signature header
+    try:
+        utils.parse_changes('6.dsc',1);
+    except utils.invalid_dsc_format_exc, line:
+        if line != 5:
+            fail("Incorrect line number ('%s') for test #6." % (line));
+    else:
+        fail("Test #6 wasn't recognised as invalid.");
+
+    # Valid .dsc ; ignoring errors
+    utils.parse_changes('1.dsc', 0);
+
+    # Invalid .dsc ; ignoring errors
+    utils.parse_changes('2.dsc', 0);
+
+################################################################################
+
+if __name__ == '__main__':
+    main()

utils.py

 # Utility functions
 # Copyright (C) 2000  James Troup <james@nocrew.org>
-# $Id: utils.py,v 1.12 2001-01-25 06:00:07 troup Exp $
+# $Id: utils.py,v 1.13 2001-01-28 09:06:44 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@ re_arch_from_filename = re.compile(r"/binary-[^/]+/")
 re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)")
 
 changes_parse_error_exc = "Can't parse line in .changes file";
+invalid_dsc_format_exc = "Invalid .dsc file";
 nk_format_exc = "Unknown Format: in .changes file";
 no_files_exc = "No Files: field in .dsc file.";
 cant_open_exc = "Can't read file.";
@@ -77,15 +78,58 @@ def extract_component_from_section(section):
 
 ######################################################################################
 
-def parse_changes(filename):
+# dsc_whitespace_rules turns on strict format checking to avoid
+# allowing in source packages which are unextracable by the
+# inappropriately fragile dpkg-source.
+#
+# The rules are:
+#
+#
+# o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
+#   followed by any PGP header data and must end with a blank line.
+#
+# o The data section must end with a blank line and must be followed by
+#   "-----BEGIN PGP SIGNATURE-----".
+
+def parse_changes(filename, dsc_whitespace_rules):
     changes_in = open_file(filename,'r');
-    error = ""
+    error = "";
     changes = {};
     lines = changes_in.readlines();
+
+    # Reindex by line number so we can easily verify the format of
+    # .dsc files...
+    index = 0;
+    indexed_lines = {};
     for line in lines:
+        index = index + 1;
+        indexed_lines[index] = line[:-1];
+
+    inside_signature = 0;
+
+    indices = indexed_lines.keys()
+    index = 0;
+    while index < max(indices):
+        index = index + 1;
+        line = indexed_lines[index];
+        if line == "":
+            if dsc_whitespace_rules:
+                index = index + 1;
+                if index > max(indices):
+                    raise invalid_dsc_format_exc, index;
+                line = indexed_lines[index];
+                if not re.match('^-----BEGIN PGP SIGNATURE', line):
+                    raise invalid_dsc_format_exc, index;
+                inside_signature = 0;
+                break;
         if re.match('^-----BEGIN PGP SIGNATURE', line):
             break;
-        if re.match(r'^\s*$|^-----BEGIN PGP SIGNED MESSAGE', line):
+        if re.match(r'^-----BEGIN PGP SIGNED MESSAGE', line):
+            if dsc_whitespace_rules:
+                inside_signature = 1;
+                while index < max(indices) and line != "":
+                    index = index + 1;
+                    line = indexed_lines[index];
             continue;
         slf = re.match(r'^(\S*)\s*:\s*(.*)', line);
         if slf:
@@ -101,10 +145,16 @@ def parse_changes(filename):
 	    changes[field] = changes[field] + mlf.groups()[0] + '\n';
             continue;
 	error = error + line;
+
+    if dsc_whitespace_rules and inside_signature:
+        raise invalid_dsc_format_exc, index;
+        
     changes_in.close();
     changes["filecontents"] = string.join (lines, "");
+
     if error != "":
 	raise changes_parse_error_exc, error;
+
     return changes;
 
 ######################################################################################