[JJ, r=JT] show-new support

ChangeLog

 2007-12-30  Joerg Jaspert  <joerg@debian.org>
 
+	* dak/dak.py (init): add show-new. This is based on a patch
+	submitted by Thomas Viehmann in Bug #408318, but large parts of
+	handling it are rewritten and show-new is done by me.
+
+	* dak/queue_report.py (table_row): Add link to generated html page
+	for NEW package.
+
+	* dak/show_new.py: new file, generates html overview for NEW
+	packages, similar to what we see with examine-package.
+
+	* config/debian/cron.hourly: Add show-new call
+
+	* config/debian/dak.conf: Add HTMLPath for Show-New
+
 	* dak/examine_package.py (print_copyright): ignore stderr when
 	finding copyright file.
+	(main): add html option
+	(html_escape): new function
+	(escape_if_needed): ditto
+	(headline): ditto
+	(colour_output): ditto
+	(print_escaped_text): ditto
+	(print_formatted_text): ditto
+	- use those functions everywhere where we generate output, as they
+	easily know if we want html or not and just DTRT
+	(do_lintian): new function
+	(check_deb): use it
+	(output_deb_info): Use print_escaped_text, not print_formatted_text.
+	Also import daklib.queue, determine_new now lives there
+
+	Also add a variable to see if we want html output. Default is
+	disabled, show_new enables it for its use.
+	Most of html, besides header/footer are in examine_package instead
+	of show_new, as it makes it a whole lot easier to deal with it at
+	the point the info is generated.
+
+
+	* dak/process_new.py (determine_new): Moved out of here.
+	(check_valid): Moved out of here.
+	(get_type): Moved out of here.
+
+	* daklib/queue.py (determine_new): Moved here.
+	(check_valid): Moved here.
+	(get_type): Moved here.
 
 	* dak/init_db.py (do_section): Remove non-US code
 

config/debian/cron.hourly

@@ -10,3 +10,4 @@ export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
 date -u > $ftpdir/project/trace/ftp-master.debian.org
 dak import-users-from-passwd
 dak queue-report -n > $webdir/new.html
+dak show-new > /dev/null

config/debian/dak.conf

@@ -89,6 +89,11 @@ Mirror-Split
   };
 };
 
+Show-New
+{
+  HTMLPath "/srv/ftp.debian.org/web/new/";
+}
+
 Import-Users-From-Passwd
 {
   ValidGID "800";

dak/dak.py

@@ -46,6 +46,8 @@ def init():
          "Archive sanity checks"),
         ("queue-report",
          "Produce a report on NEW and BYHAND packages"),
+        ("show-new",
+         "Output html for packages in NEW"),
         
         ("rm",
          "Remove packages from suites"),

dak/examine_package.py

@@ -34,7 +34,7 @@
 
 import errno, os, pg, re, sys, md5
 import apt_pkg, apt_inst
-import daklib.database, daklib.utils
+import daklib.database, daklib.utils, daklib.queue
 
 ################################################################################
 
@@ -54,24 +54,8 @@ re_version = re.compile('^(.*)\((.*)\)')
 re_newlinespace = re.compile('\n')
 re_spacestrip = re.compile('(\s)')
 
-################################################################################
-
-# Colour definitions
-
-# Main
-main_colour = "\033[36m"
-# Contrib
-contrib_colour = "\033[33m"
-# Non-Free
-nonfree_colour = "\033[31m"
-# Arch
-arch_colour = "\033[32m"
-# End
-end_colour = "\033[0m"
-# Bold
-bold_colour = "\033[1m"
-# Bad maintainer
-maintainer_colour = arch_colour
+html_escaping = {'"':'&quot;', '&':'&amp;', '<':'&lt;', '>':'&gt;'}
+re_html_escaping = re.compile('|'.join(map(re.escape, html_escaping.keys())))
 
 ################################################################################
 
@@ -84,6 +68,9 @@ daklib.database.init(Cnf, projectB)
 
 printed_copyrights = {}
 
+# default is to not output html.
+use_html = 0
+
 ################################################################################
 
 def usage (exit_code=0):
@@ -91,11 +78,68 @@ def usage (exit_code=0):
 Check NEW package(s).
 
   -h, --help                 show this help and exit
+  -H, --html-output          output html page with inspection result
+  -f, --file-name            filename for the html page
 
 PACKAGE can be a .changes, .dsc, .deb or .udeb filename."""
 
     sys.exit(exit_code)
 
+################################################################################
+# probably xml.sax.saxutils would work as well
+
+def html_escape(s):
+  return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)
+
+def escape_if_needed(s):
+  if use_html:
+    return re_html_escaping.sub(html_escaping.get, s)
+  else:
+    return s
+  
+def headline(s, level=2):
+  if use_html:
+    print "<h%d>%s</h%d>" % (level, html_escape(s), level)
+  else:
+    print "---- %s ----" % (s)
+
+# Colour definitions, 'end' isn't really for use
+
+ansi_colours = {
+  'main': "\033[36m",
+  'contrib': "\033[33m",
+  'nonfree': "\033[31m",
+  'arch': "\033[32m",
+  'end': "\033[0m",
+  'bold': "\033[1m",
+  'maintainer': "\033[32m"}
+
+html_colours = {
+  'main': ('<span style="color: aqua">',"</span>"),
+  'contrib': ('<span style="color: yellow">',"</span>"),
+  'nonfree': ('<span style="color: red">',"</span>"),
+  'arch': ('<span style="color: green">',"</span>"),
+  'bold': ('<span style="font-weight: bold">',"</span>"),
+  'maintainer': ('<span style="color: green">',"</span>")}
+
+def colour_output(s, colour):
+  if use_html:
+    return ("%s%s%s" % (html_colours[colour][0], html_escape(s), html_colours[colour][1]))
+  else:
+    return ("%s%s%s" % (ansi_colours[colour], s, ansi_colours['end']))
+
+def print_escaped_text(s):
+  if use_html:
+    print "<pre>%s</pre>" % (s)
+  else:
+    print s  
+
+def print_formatted_text(s):
+  if use_html:
+    print "<pre>%s</pre>" % (html_escape(s))
+  else:
+    print s
+
 ################################################################################
 
 def get_depends_parts(depend) :
@@ -150,7 +194,8 @@ def read_control (filename):
 	extracts = apt_inst.debExtractControl(deb_file)
 	control = apt_pkg.ParseSection(extracts)
     except:
-	print "can't parse control info"
+	print_formatted_text("can't parse control info")
+	# TV-COMMENT: this will raise exceptions in two lines
 	control = ''
 
     deb_file.close()
@@ -173,23 +218,25 @@ def read_control (filename):
 	nf_match = re_nonfree.search(section_str)
 	if c_match :
 	    # contrib colour
-	    section = contrib_colour + section_str + end_colour
+	    section = colour_output(section_str, 'contrib')
 	elif nf_match :
 	    # non-free colour
-	    section = nonfree_colour + section_str + end_colour
+	    section = colour_output(section_str, 'nonfree')
 	else :
 	    # main
-	    section = main_colour +  section_str + end_colour
+	    section = colour_output(section_str, 'main')
     if control.has_key("Architecture"):
 	arch_str = control.Find("Architecture")
-   	arch = arch_colour + arch_str + end_colour
+   	arch = colour_output(arch_str, 'arch')
 
     if control.has_key("Maintainer"):
 	maintainer = control.Find("Maintainer")
    	localhost = re_localhost.search(maintainer)
 	if localhost:
 	    #highlight bad email
-	    maintainer = maintainer_colour + maintainer + end_colour
+	    maintainer = colour_output(maintainer, 'maintainer')
+	else:
+	    maintainer = escape_if_needed(maintainer)
 
     return (control, control_keys, section, depends, recommends, arch, maintainer)
 
@@ -200,10 +247,10 @@ def read_dsc (dsc_filename):
     try:
 	dsc = daklib.utils.parse_changes(dsc_filename)
     except:
-	print "can't parse control info"
+	print_formatted_text("can't parse control info")
     dsc_file.close()
 
-    filecontents = strip_pgp_signature(dsc_filename)
+    filecontents = escape_if_needed(strip_pgp_signature(dsc_filename))
 
     if dsc.has_key("build-depends"):
 	builddep = split_depends(dsc["build-depends"])
@@ -216,7 +263,7 @@ def read_dsc (dsc_filename):
 
     if dsc.has_key("architecture") :
 	if (dsc["architecture"] != "any"):
-	    newarch = arch_colour + dsc["architecture"] + end_colour
+	    newarch = colour_output(dsc["architecture"], 'arch')
 	    filecontents = re_arch.sub("Architecture: " + newarch, filecontents)
 
     return filecontents
@@ -240,21 +287,21 @@ def create_depends_string (depends_tree):
 	    if ql:
 		i = ql[0]
 
+		adepends = d['name']
+		if d['version'] != '' :
+		    adepends += " (%s)" % (d['version'])
+		
 		if i[2] == "contrib":
-		    result += contrib_colour + d['name']
+		    result += colour_output(adepends, "contrib")
 		elif i[2] == "non-free":
-		    result += nonfree_colour + d['name']
+		    result += colour_output(adepends, "nonfree")
 		else :
-		    result += main_colour + d['name']
-
-		if d['version'] != '' :
-		    result += " (%s)" % (d['version'])
-		result += end_colour
+		    result += colour_output(adepends, "main")
 	    else:
-		result += bold_colour + d['name']
+		adepends = d['name']
 		if d['version'] != '' :
-		    result += " (%s)" % (d['version'])
-		result += end_colour
+		    adepends += " (%s)" % (d['version'])
+		result += colour_output(adepends, "bold")
 	    or_count += 1
 	comma_count += 1
     return result
@@ -262,8 +309,9 @@ def create_depends_string (depends_tree):
 def output_deb_info(filename):
     (control, control_keys, section, depends, recommends, arch, maintainer) = read_control(filename)
 
+    to_print = ""
     if control == '':
-	print "no control info"
+	print_formatted_text("no control info")
     else:
 	for key in control_keys :
 	    output = " " + key + ": "
@@ -280,14 +328,22 @@ def output_deb_info(filename):
 	    elif key == 'Description':
 		desc = control.Find(key)
 		desc = re_newlinespace.sub('\n ', desc)
-		output += desc
+		output += escape_if_needed(desc)
 	    else:
-		output += control.Find(key)
-	    print output
+		output += escape_if_needed(control.Find(key))
+            to_print += output + '\n'
+        print_escaped_text(to_print)
 
 def do_command (command, filename):
     o = os.popen("%s %s" % (command, filename))
-    print o.read()
+    print_formatted_text(o.read())
+
+def do_lintian (filename):
+    # lintian currently does not have html coloring, so dont use color for lintian (yet)
+    if use_html:
+        do_command("lintian --show-overrides", filename)
+    else:
+        do_command("lintian --show-overrides --color always", filename)
 
 def print_copyright (deb_filename):
     package = re_package.sub(r'\1', deb_filename)
@@ -295,12 +351,12 @@ def print_copyright (deb_filename):
     copyright = o.read()[:-1]
 
     if copyright == "":
-        print "WARNING: No copyright found, please check package manually."
+        print_formatted_text("WARNING: No copyright found, please check package manually.")
         return
 
     doc_directory = re_doc_directory.sub(r'\1', copyright)
     if package != doc_directory:
-        print "WARNING: wrong doc directory (expected %s, got %s)." % (package, doc_directory)
+        print_formatted_text("WARNING: wrong doc directory (expected %s, got %s)." % (package, doc_directory))
         return
 
     o = os.popen("dpkg-deb --fsys-tarfile %s | tar xvOf - %s 2>/dev/null" % (deb_filename, copyright))
@@ -308,19 +364,19 @@ def print_copyright (deb_filename):
     copyrightmd5 = md5.md5(copyright).hexdigest()
 
     if printed_copyrights.has_key(copyrightmd5) and printed_copyrights[copyrightmd5] != "%s (%s)" % (package, deb_filename):
-        print "NOTE: Copyright is the same as %s.\n" % \
-		(printed_copyrights[copyrightmd5])
+        print_formatted_text( "NOTE: Copyright is the same as %s.\n" % \
+		(printed_copyrights[copyrightmd5]))
     else:
 	printed_copyrights[copyrightmd5] = "%s (%s)" % (package, deb_filename)
 
-    print copyright
+    print_formatted_text(copyright)
 
 def check_dsc (dsc_filename):
-    print "---- .dsc file for %s ----" % (dsc_filename)
+    headline(".dsc file for %s" % (dsc_filename))
     (dsc) = read_dsc(dsc_filename)
-    print dsc
-    print "---- lintian check for %s ----" % (dsc_filename)
-    do_command("lintian --show-overrides --color always", dsc_filename)
+    print_escaped_text(dsc)
+    headline("lintian check for %s" % (dsc_filename))
+    do_lintian(dsc_filename)
 
 def check_deb (deb_filename):
     filename = os.path.basename(deb_filename)
@@ -330,29 +386,29 @@ def check_deb (deb_filename):
     else:
 	is_a_udeb = 0
 
-    print "---- control file for %s ----" % (filename)
+    headline("control file for %s" % (filename))
     #do_command ("dpkg -I", deb_filename)
     output_deb_info(deb_filename)
 
     if is_a_udeb:
-	print "---- skipping lintian check for udeb ----"
+	headline("skipping lintian check for udeb")
 	print 
     else:
-	print "---- lintian check for %s ----" % (filename)
-        do_command ("lintian --show-overrides --color always", deb_filename)
-	print "---- linda check for %s ----" % (filename)
+	headline("lintian check for %s" % (filename))
+        do_lintian(deb_filename)
+	headline("---- linda check for %s ----" % (filename))
         do_command ("linda", deb_filename)
 
-    print "---- contents of %s ----" % (filename)
+    headline("contents of %s" % (filename))
     do_command ("dpkg -c", deb_filename)
 
     if is_a_udeb:
-	print "---- skipping copyright for udeb ----"
+	headline("skipping copyright for udeb")
     else:
-	print "---- copyright of %s ----" % (filename)
+	headline("copyright of %s" % (filename))
         print_copyright(deb_filename)
 
-    print "---- file listing of %s ----" % (filename)
+    headline("file listing of %s" % (filename))
     do_command ("ls -l", deb_filename)
 
 # Read a file, strip the signature and return the modified contents as
@@ -385,8 +441,8 @@ def strip_pgp_signature (filename):
 
 # Display the .changes [without the signature]
 def display_changes (changes_filename):
-    print "---- .changes file for %s ----" % (changes_filename)
-    print strip_pgp_signature(changes_filename)
+    headline(".changes file for %s" % (changes_filename))
+    print_formatted_text(strip_pgp_signature(changes_filename))
 
 def check_changes (changes_filename):
     display_changes(changes_filename)
@@ -405,9 +461,11 @@ def main ():
 
 #    Cnf = daklib.utils.get_conf()
 
-    Arguments = [('h',"help","Examine-Package::Options::Help")]
-    for i in [ "help" ]:
-	if not Cnf.has_key("Frenanda::Options::%s" % (i)):
+    Arguments = [('h',"help","Examine-Package::Options::Help"),
+                 ('H',"html-output","Examine-Package::Options::Html-Output"),
+                ]
+    for i in [ "Help", "Html-Output", "partial-html" ]:
+	if not Cnf.has_key("Examine-Package::Options::%s" % (i)):
 	    Cnf["Examine-Package::Options::%s" % (i)] = ""
 
     args = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
@@ -420,11 +478,11 @@ def main ():
 
     for file in args:
         try:
-            # Pipe output for each argument through less
-            less_fd = os.popen("less -R -", 'w', 0)
-	    # -R added to display raw control chars for colour
-            sys.stdout = less_fd
-
+	    if not Options["Html-Output"]:
+		# Pipe output for each argument through less
+		less_fd = os.popen("less -R -", 'w', 0)
+		# -R added to display raw control chars for colour
+		sys.stdout = less_fd
             try:
                 if file.endswith(".changes"):
                     check_changes(file)
@@ -435,9 +493,10 @@ def main ():
                 else:
                     daklib.utils.fubar("Unrecognised file type: '%s'." % (file))
             finally:
-                # Reset stdout here so future less invocations aren't FUBAR
-                less_fd.close()
-                sys.stdout = stdout_fd
+		if not Options["Html-Output"]:
+		    # Reset stdout here so future less invocations aren't FUBAR
+		    less_fd.close()
+		    sys.stdout = stdout_fd
         except IOError, e:
             if errno.errorcode[e.errno] == 'EPIPE':
                 daklib.utils.warn("[examine-package] Caught EPIPE; skipping.")

dak/process_new.py

@@ -122,71 +122,6 @@ def recheck():
 
 ################################################################################
 
-def determine_new (changes, files):
-    new = {}
-
-    # Build up a list of potentially new things
-    for file in files.keys():
-        f = files[file]
-        # Skip byhand elements
-        if f["type"] == "byhand":
-            continue
-        pkg = f["package"]
-        priority = f["priority"]
-        section = f["section"]
-        # FIXME: unhardcode
-        if section == "non-US/main":
-            section = "non-US"
-        type = get_type(f)
-        component = f["component"]
-
-        if type == "dsc":
-            priority = "source"
-        if not new.has_key(pkg):
-            new[pkg] = {}
-            new[pkg]["priority"] = priority
-            new[pkg]["section"] = section
-            new[pkg]["type"] = type
-            new[pkg]["component"] = component
-            new[pkg]["files"] = []
-        else:
-            old_type = new[pkg]["type"]
-            if old_type != type:
-                # source gets trumped by deb or udeb
-                if old_type == "dsc":
-                    new[pkg]["priority"] = priority
-                    new[pkg]["section"] = section
-                    new[pkg]["type"] = type
-                    new[pkg]["component"] = component
-        new[pkg]["files"].append(file)
-        if f.has_key("othercomponents"):
-            new[pkg]["othercomponents"] = f["othercomponents"]
-
-    for suite in changes["suite"].keys():
-        suite_id = daklib.database.get_suite_id(suite)
-        for pkg in new.keys():
-            component_id = daklib.database.get_component_id(new[pkg]["component"])
-            type_id = daklib.database.get_override_type_id(new[pkg]["type"])
-            q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
-            ql = q.getresult()
-            if ql:
-                for file in new[pkg]["files"]:
-                    if files[file].has_key("new"):
-                        del files[file]["new"]
-                del new[pkg]
-
-    if changes["suite"].has_key("stable"):
-        print "WARNING: overrides will be added for stable!"
-    if changes["suite"].has_key("oldstable"):
-        print "WARNING: overrides will be added for OLDstable!"
-    for pkg in new.keys():
-        if new[pkg].has_key("othercomponents"):
-            print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
-
-    return new
-
-################################################################################
-
 def indiv_sg_compare (a, b):
     """Sort by source name, source, version, 'have source', and
        finally by filename."""
@@ -320,25 +255,8 @@ class Priority_Completer:
 
 ################################################################################
 
-def check_valid (new):
-    for pkg in new.keys():
-        section = new[pkg]["section"]
-        priority = new[pkg]["priority"]
-        type = new[pkg]["type"]
-        new[pkg]["section id"] = daklib.database.get_section_id(section)
-        new[pkg]["priority id"] = daklib.database.get_priority_id(new[pkg]["priority"])
-        # Sanity checks
-        di = section.find("debian-installer") != -1
-        if (di and type != "udeb") or (not di and type == "udeb"):
-            new[pkg]["section id"] = -1
-        if (priority == "source" and type != "dsc") or \
-           (priority != "source" and type == "dsc"):
-            new[pkg]["priority id"] = -1
-
-################################################################################
-
 def print_new (new, indexed, file=sys.stdout):
-    check_valid(new)
+    daklib.queue.check_valid(new)
     broken = 0
     index = 0
     for pkg in new.keys():
@@ -366,24 +284,6 @@ def print_new (new, indexed, file=sys.stdout):
 
 ################################################################################
 
-def get_type (f):
-    # Determine the type
-    if f.has_key("dbtype"):
-        type = f["dbtype"]
-    elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
-        type = "dsc"
-    else:
-        daklib.utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (type))
-
-    # Validate the override type
-    type_id = daklib.database.get_override_type_id(type)
-    if type_id == -1:
-        daklib.utils.fubar("invalid type (%s) for new.  Say wha?" % (type))
-
-    return type
-
-################################################################################
-
 def index_range (index):
     if index == 1:
         return "1"
@@ -721,7 +621,7 @@ def do_new():
     done = 0
     while not done:
         # Find out what's new
-        new = determine_new(changes, files)
+        new = daklib.queue.determine_new(changes, files, projectB)
 
         if not new:
             break

dak/queue_report.py

@@ -239,7 +239,7 @@ def table_row(source, version, arch, last_mod, maint, distribution, closes):
     print "<td valign=\"top\" class=\"%s\">%s</td>" % (tdclass, source)
     print "<td valign=\"top\" class=\"%s\">" % (tdclass)
     for vers in version.split():
-        print "%s<br>" % (vers)
+        print "<a href=\"/new/%s_%s.html\">%s</a><br>" % (source, vers, vers)
     print "</td><td valign=\"top\" class=\"%s\">%s</td><td valign=\"top\" class=\"%s\">" % (tdclass, arch, tdclass)
     for dist in distribution:
         print "%s<br>" % (dist)

dak/show_new.py

+#!/usr/bin/env python
+
+# Output html for packages in NEW
+# Copyright (C) 2007 Joerg Jaspert <joerg@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+################################################################################
+
+# <elmo> I'm James Troup, long term source of all evil in Debian. you may
+#        know me from such debian-devel-announce gems as "Serious
+#        Problems With ...."
+
+################################################################################
+
+import copy, os, sys, time
+import apt_pkg
+import examine_package
+import daklib.database
+import daklib.queue 
+import daklib.utils
+
+# Globals
+Cnf = None
+Options = None
+Upload = None
+projectB = None
+sources = set()
+
+
+################################################################################
+################################################################################
+################################################################################
+
+def html_header(name):
+    if name.endswith('.changes'):
+        name = ' '.join(name.split('_')[:2])
+    print """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+        <html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8">"""
+    print "<title>%s - Debian NEW package overview</title>" % (name)
+    print """<link type="text/css" rel="stylesheet" href="/style.css">
+	<link rel="shortcut icon" href="http://www.debian.org/favicon.ico">
+	</head>
+	<body>
+	<div align="center">
+	<a href="http://www.debian.org/">
+     <img src="http://www.debian.org/logos/openlogo-nd-50.png" border="0" hspace="0" vspace="0" alt=""></a>
+	<a href="http://www.debian.org/">
+     <img src="http://www.debian.org/Pics/debian.png" border="0" hspace="0" vspace="0" alt="Debian Project"></a>
+	</div>
+	<br />
+	<table class="reddy" width="100%">
+	<tr>
+	<td class="reddy">
+    <img src="http://www.debian.org/Pics/red-upperleft.png" align="left" border="0" hspace="0" vspace="0"
+     alt="" width="15" height="16"></td>"""
+    print """<td rowspan="2" class="reddy">Debian NEW package overview for %s</td>""" % (name)
+    print """<td class="reddy">
+    <img src="http://www.debian.org/Pics/red-upperright.png" align="right" border="0" hspace="0" vspace="0"
+     alt="" width="16" height="16"></td>
+	</tr>
+	<tr>
+	<td class="reddy">
+    <img src="http://www.debian.org/Pics/red-lowerleft.png" align="left" border="0" hspace="0" vspace="0"
+     alt="" width="16" height="16"></td>
+	<td class="reddy">
+    <img src="http://www.debian.org/Pics/red-lowerright.png" align="right" border="0" hspace="0" vspace="0"
+     alt="" width="15" height="16"></td>
+	</tr>
+	</table>
+	"""
+
+def html_footer():
+    print "<p class=\"validate\">Timestamp: %s (UTC)</p>" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime()))
+    print """<a href="http://validator.w3.org/check?uri=referer">
+    <img border="0" src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88"></a>
+	<a href="http://jigsaw.w3.org/css-validator/check/referer">
+    <img border="0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!"
+     height="31" width="88"></a>
+    """
+    print "</body></html>"
+
+
+################################################################################
+
+
+def do_pkg(changes_file):
+    Upload.pkg.changes_file = changes_file
+    Upload.init_vars()
+    Upload.update_vars()
+    files = Upload.pkg.files
+    changes = Upload.pkg.changes
+
+    changes["suite"] = copy.copy(changes["distribution"])
+
+    # Find out what's new
+    new = daklib.queue.determine_new(changes, files, projectB, 0)
+
+    stdout_fd = sys.stdout
+
+    htmlname = changes["source"] + "_" + changes["version"] + ".html"
+    sources.add(htmlname)
+    # do not generate html output if that source/version already has one.
+    if not os.path.exists(os.path.join(Cnf["Show-New::HTMLPath"],htmlname)):
+        sys.stdout = open(os.path.join(Cnf["Show-New::HTMLPath"],htmlname),"w")
+        html_header(changes["source"])
+
+        daklib.queue.check_valid(new)
+        examine_package.display_changes(Upload.pkg.changes_file)
+
+        for pkg in new.keys():
+            for file in new[pkg]["files"]:
+                if ( files[file].has_key("new") and not
+                     files[file]["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2"] ):
+                    if file.endswith(".deb") or file.endswith(".udeb"):
+                        examine_package.check_deb(file)
+                    elif file.endswith(".dsc"):
+                        examine_package.check_dsc(file)
+
+        html_footer()
+        if sys.stdout != stdout_fd:
+            sys.stdout.close()
+            sys.stdout = stdout_fd
+
+################################################################################
+
+def usage (exit_code=0):
+    print """Usage: dak show-new [OPTION]... [CHANGES]...
+  -h, --help                show this help and exit.
+  """
+    sys.exit(exit_code)
+
+################################################################################
+
+def init():
+    global Cnf, Options, Upload, projectB
+
+    Cnf = daklib.utils.get_conf()
+
+    Arguments = [('h',"help","Show-New::Options::Help")]
+
+    for i in ["help"]:
+        if not Cnf.has_key("Show-New::Options::%s" % (i)):
+            Cnf["Show-New::Options::%s" % (i)] = ""
+
+    changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
+    Options = Cnf.SubTree("Show-New::Options")
+
+    if Options["help"]:
+        usage()
+
+    Upload = daklib.queue.Upload(Cnf)
+
+    projectB = Upload.projectB
+
+    return changes_files
+
+
+################################################################################
+################################################################################
+
+def main():
+    changes_files = init()
+
+    examine_package.use_html=1
+
+    for changes_file in changes_files:
+        changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
+        if not changes_file:
+            continue
+        print "\n" + changes_file
+        do_pkg (changes_file)
+    files = set(os.listdir(Cnf["Show-New::HTMLPath"]))
+    to_delete = files.difference(sources)
+    for file in to_delete:
+        os.remove(os.path.join(Cnf["Show-New::HTMLPath"],file))
+
+################################################################################
+
+if __name__ == '__main__':
+    main()

daklib/queue.py

@@ -32,6 +32,109 @@ re_default_answer = re.compile(r"\[(.*)\]")
 re_fdnic = re.compile(r"\n\n")
 re_bin_only_nmu = re.compile(r"\+b\d+$")
 
+################################################################################
+
+# Determine what parts in a .changes are NEW
+
+def determine_new(changes, files, projectB, warn=1):
+    new = {}
+
+    # Build up a list of potentially new things
+    for file in files.keys():
+        f = files[file]
+        # Skip byhand elements
+        if f["type"] == "byhand":
+            continue
+        pkg = f["package"]
+        priority = f["priority"]
+        section = f["section"]
+        type = get_type(f)
+        component = f["component"]
+
+        if type == "dsc":
+            priority = "source"
+        if not new.has_key(pkg):
+            new[pkg] = {}
+            new[pkg]["priority"] = priority
+            new[pkg]["section"] = section
+            new[pkg]["type"] = type
+            new[pkg]["component"] = component
+            new[pkg]["files"] = []
+        else:
+            old_type = new[pkg]["type"]
+            if old_type != type:
+                # source gets trumped by deb or udeb
+                if old_type == "dsc":
+                    new[pkg]["priority"] = priority
+                    new[pkg]["section"] = section
+                    new[pkg]["type"] = type
+                    new[pkg]["component"] = component
+        new[pkg]["files"].append(file)
+        if f.has_key("othercomponents"):
+            new[pkg]["othercomponents"] = f["othercomponents"]
+
+    for suite in changes["suite"].keys():
+        suite_id = database.get_suite_id(suite)
+        for pkg in new.keys():
+            component_id = database.get_component_id(new[pkg]["component"])
+            type_id = database.get_override_type_id(new[pkg]["type"])
+            q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
+            ql = q.getresult()
+            if ql:
+                for file in new[pkg]["files"]:
+                    if files[file].has_key("new"):
+                        del files[file]["new"]
+                del new[pkg]
+
+    if warn:
+        if changes["suite"].has_key("stable"):
+            print "WARNING: overrides will be added for stable!"
+            if changes["suite"].has_key("oldstable"):
+                print "WARNING: overrides will be added for OLDstable!"
+        for pkg in new.keys():
+            if new[pkg].has_key("othercomponents"):
+                print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
+
+    return new
+
+################################################################################
+
+def get_type(f):
+    # Determine the type
+    if f.has_key("dbtype"):
+        type = f["dbtype"]
+    elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
+        type = "dsc"
+    else:
+        fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (type))
+
+    # Validate the override type
+    type_id = database.get_override_type_id(type)
+    if type_id == -1:
+        fubar("invalid type (%s) for new.  Say wha?" % (type))
+
+    return type
+
+################################################################################
+
+# check if section/priority values are valid
+
+def check_valid(new):
+    for pkg in new.keys():
+        section = new[pkg]["section"]
+        priority = new[pkg]["priority"]
+        type = new[pkg]["type"]
+        new[pkg]["section id"] = database.get_section_id(section)
+        new[pkg]["priority id"] = database.get_priority_id(new[pkg]["priority"])
+        # Sanity checks
+        di = section.find("debian-installer") != -1
+        if (di and type != "udeb") or (not di and type == "udeb"):
+            new[pkg]["section id"] = -1
+        if (priority == "source" and type != "dsc") or \
+           (priority != "source" and type == "dsc"):
+            new[pkg]["priority id"] = -1
+
+
 ###############################################################################
 
 # Convenience wrapper to carry around all the package information in