changes for DM support

config/debian/cron.daily

@@ -169,6 +169,8 @@ dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-mast
 # and one on crufty packages
 dak cruft-report | tee $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
 
+$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
+
 ################################################################################
 
 # Run mirror-split

config/debian/dak.conf

@@ -3,6 +3,7 @@ Dinstall
    GPGKeyring {
       "/srv/keyring.debian.org/keyrings/debian-keyring.gpg"; 
       "/srv/keyring.debian.org/keyrings/debian-keyring.pgp";
+      "/srv/ftp.debian.org/keyrings/debian-maintainers.gpg";
    };
    SigningKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg";
    SigningPubKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg";
@@ -636,6 +637,13 @@ AutomaticByHandPackages {
     Script "/srv/ftp.debian.org/dak/scripts/debian/byhand-di";
   };
 
+  "debian-maintainers" {
+    Source "debian-maintainers";
+    Section "raw-keyring";
+    Extension "gpg";
+    Script "/srv/ftp.debian.org/dak/scripts/debian/byhand-dm";
+  };
+
   "tag-overrides" {
     Source "tag-overrides";
     Section "byhand";

dak/clean_suites.py

@@ -199,6 +199,7 @@ def clean():
         before = time.time()
         sys.stdout.write("[Deleting from source table... ")
         projectB.query("DELETE FROM dsc_files WHERE EXISTS (SELECT 1 FROM source s, files f, dsc_files df WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = df.source AND df.id = dsc_files.id)" % (delete_date))
+        projectB.query("DELETE FROM src_uploaders WHERE EXISTS (SELECT 1 FROM source s, files f WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = src_uploaders.source)" % (delete_date))
         projectB.query("DELETE FROM source WHERE EXISTS (SELECT 1 FROM files WHERE source.file = files.id AND files.last_used <= '%s')" % (delete_date))
         sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before)))
 
@@ -249,7 +250,8 @@ def clean_maintainers():
     q = projectB.query("""
 SELECT m.id FROM maintainer m
   WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.maintainer = m.id)
-    AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id)""")
+    AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id)
+    AND NOT EXISTS (SELECT 1 FROM src_uploaders u WHERE u.maintainer = m.id)""")
     ql = q.getresult()
 
     count = 0
@@ -271,7 +273,8 @@ def clean_fingerprints():
 
     q = projectB.query("""
 SELECT f.id FROM fingerprint f
-  WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id)
+  WHERE f.keyring IS NULL
+    AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id)
     AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)""")
     ql = q.getresult()
 

dak/dak.py

@@ -90,6 +90,8 @@ def init():
          "Check for users with no packages in the archive"),
         ("import-archive",
          "Populate SQL database based from an archive tree"),
+        ("import-keyring",
+         "Populate fingerprint/uid table based on a new/updated keyring"),
         ("import-ldap-fingerprints",
          "Syncs fingerprint and uid tables with Debian LDAP db"),
         ("import-users-from-passwd",

dak/import_ldap_fingerprints.py

@@ -149,9 +149,13 @@ SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id
                 if not existing_uid:
                     q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
                     print "Assigning %s to 0x%s." % (uid, fingerprint)
+		elif existing_uid == uid:
+		    pass
+		elif existing_uid[:3] == "dm:":
+                    q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
+                    print "Promoting DM %s to DD %s with keyid 0x%s." % (existing_uid, uid, fingerprint)
                 else:
-                    if existing_uid != uid:
-                        daklib.utils.fubar("%s has %s in LDAP, but projectB says it should be %s." % (uid, fingerprint, existing_uid))
+                    daklib.utils.warn("%s has %s in LDAP, but projectB says it should be %s." % (uid, fingerprint, existing_uid))
 
     # Try to update people who sign with non-primary key
     q = projectB.query("SELECT fingerprint, id FROM fingerprint WHERE uid is null")
@@ -168,10 +172,11 @@ SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id
             primary_key = m.group(1)
             primary_key = primary_key.replace(" ","")
             if not ldap_fin_uid_id.has_key(primary_key):
-                daklib.utils.fubar("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint))
-            (uid, uid_id) = ldap_fin_uid_id[primary_key]
-            q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
-            print "Assigning %s to 0x%s." % (uid, fingerprint)
+                daklib.utils.warn("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint))
+	    else:
+            	(uid, uid_id) = ldap_fin_uid_id[primary_key]
+            	q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
+            	print "Assigning %s to 0x%s." % (uid, fingerprint)
         else:
             extra_keyrings = ""
             for keyring in Cnf.ValueList("Import-LDAP-Fingerprints::ExtraKeyrings"):
@@ -204,6 +209,7 @@ SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id
                 guess_uid = "???"
             name = " ".join(output.split('\n')[0].split()[3:])
             print "0x%s -> %s -> %s" % (fingerprint, name, guess_uid)
+
             # FIXME: make me optionally non-interactive
             # FIXME: default to the guessed ID
             uid = None

dak/process_accepted.py

@@ -309,6 +309,19 @@ def install ():
                     files_id = daklib.database.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id)
                 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id))
 
+            # Add the src_uploaders to the DB
+            if dsc.get("dm-upload-allowed", "no") == "yes":
+                uploader_ids = []
+                if dsc.has_key("uploaders"):
+                    uploader_ids = [
+                        daklib.database.get_or_set_maintainer_id( u.strip() )
+                          for u in dsc["uploaders"].split(",")
+                    ]
+                uploader_ids.append(maintainer_id)
+                for u in uploader_ids:
+                    projectB.query("INSERT INTO src_uploaders (source, maintainer) VALUES (currval('source_id_seq'), %d)" % (u))
+
+
     # Add the .deb files to the DB
     for file in files.keys():
         if files[file]["type"] == "deb":

dak/process_unchecked.py

@@ -1016,10 +1016,84 @@ def check_timestamps():
 
 ################################################################################
 
+def lookup_uid_from_fingerprint(fpr):
+    q = Upload.projectB.query("SELECT u.uid, u.name FROM fingerprint f, uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr))
+    qs = q.getresult()
+    if len(qs) == 0:
+        return (None, None)
+    else:
+        return qs[0]
+
 def check_signed_by_key():
     """Ensure the .changes is signed by an authorized uploader."""
 
-    # We only check binary-only uploads right now
+    (uid, uid_name) = lookup_uid_from_fingerprint(changes["fingerprint"])
+    if uid_name == None:
+        uid_name = ""
+
+    # match claimed name with actual name:
+    if uid == None:
+        uid, uid_email = changes["fingerprint"], uid
+        may_nmu, may_sponsor = 1, 1
+	# XXX by default new dds don't have a fingerprint/uid in the db atm,
+	#     and can't get one in there if we don't allow nmu/sponsorship
+    elif uid[:3] == "dm:":
+        uid_email = uid[3:]
+        may_nmu, may_sponsor = 0, 0
+    else:
+        uid_email = "%s@debian.org" % (uid)
+        may_nmu, may_sponsor = 1, 1
+
+    if uid_email in [changes["maintaineremail"], changes["changedbyemail"]]:
+        sponsored = 0
+    elif uid_name in [changes["maintainername"], changes["changedbyname"]]:
+        sponsored = 0
+        if uid_name == "": sponsored = 1
+    else:
+        sponsored = 1
+
+    if sponsored and not may_sponsor: 
+        reject("%s is not authorised to sponsor uploads" % (uid))
+
+    if not sponsored and not may_nmu:
+        source_ids = []
+	check_suites = changes["distribution"].keys()
+	if "unstable" not in check_suites: check_suites.append("unstable")
+        for suite in check_suites:
+            suite_id = daklib.database.get_suite_id(suite)
+            q = Upload.projectB.query("SELECT s.id FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND sa.suite = %d" % (changes["source"], suite_id))
+            for si in q.getresult():
+                if si[0] not in source_ids: source_ids.append(si[0])
+
+        print "source_ids: %s" % (",".join([str(x) for x in source_ids]))
+
+        is_nmu = 1
+        for si in source_ids:
+            is_nmu = 1
+            q = Upload.projectB.query("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT maintainer FROM src_uploaders WHERE src_uploaders.source = %s)" % (si))
+            for m in q.getresult():
+                (rfc822, rfc2047, name, email) = daklib.utils.fix_maintainer(m[0])
+                if email == uid_email or name == uid_name:
+                    is_nmu=0
+                    break
+        if is_nmu:
+            reject("%s may not upload/NMU source package %s" % (uid, changes["source"]))
+
+        for b in changes["binary"].keys():
+            for suite in changes["distribution"].keys():
+                suite_id = daklib.database.get_suite_id(suite)
+	        q = Upload.projectB.query("SELECT DISTINCT s.source FROM source s JOIN binaries b ON (s.id = b.source) JOIN bin_associations ba On (b.id = ba.bin) WHERE b.package = '%s' AND ba.suite = %s" % (b, suite_id))
+		for s in q.getresult():
+                    if s[0] != changes["source"]:
+                        reject("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite))
+
+        for file in files.keys():
+            if files[file].has_key("byhand"): 
+                reject("%s may not upload BYHAND file %s" % (uid, file))
+            if files[file].has_key("new"):
+                reject("%s may not upload NEW file %s" % (uid, file))
+
+    # The remaining checks only apply to binary-only uploads right now
     if changes["architecture"].has_key("source"):
         return
 

daklib/queue.py

@@ -134,7 +134,7 @@ class Upload:
                 d_changes[i] = changes[i]
         ## dsc
         for i in [ "source", "version", "maintainer", "fingerprint",
-                   "uploaders", "bts changelog" ]:
+                   "uploaders", "bts changelog", "dm-upload-allowed" ]:
             if dsc.has_key(i):
                 d_dsc[i] = dsc[i]
         ## dsc_files

scripts/debian/byhand-dm

+#!/bin/sh -e
+
+BYHAND="$1"
+VERSION="$2"
+ARCH="$3"
+CHANGES="$4"
+
+KEYRING=/srv/keyring.debian.org/keyrings/debian-keyring.gpg
+
+DESTKR=/srv/ftp.debian.org/keyrings/debian-maintainers.gpg
+
+get_id () {
+  echo "SELECT U.name, U.uid FROM fingerprint F JOIN uid U ON (F.uid = U.id) WHERE F.fingerprint = '$1';" |
+    psql projectb -At |
+    sed 's/|\(.*\)/ <\1@debian.org>/'
+}
+
+is_allowed () {
+  echo "SELECT M.name from src_uploaders U join source S on (U.source = S.id) join maintainer M on (U.maintainer = M.id) WHERE S.source = 'debian-maintainers';" |
+    psql projectb -At | 
+    while read ALLOWED; do
+      if [ "$1" = "$ALLOWED" ]; then
+        echo yes
+	break
+      fi
+    done
+}
+
+FPRINT=$(gpgv --keyring "$KEYRING" --status-fd 3 3>&1 >/dev/null 2>&1 "$CHANGES" |
+    cut -d\  -f2,3 | grep ^VALIDSIG | head -n1 | cut -d\  -f2)
+
+ID="$(get_id "$FPRINT")"
+
+if [ "$(is_allowed "$ID")" != "yes" ]; then
+  echo "Unauthorised upload by $ID"
+  exit 1
+fi
+
+echo "Authorised upload by $ID, copying into place"
+
+OUT=$(mktemp)
+
+cp "$BYHAND" "$DESTKR"
+dak import-keyring --generate-users "dm:%s" "$DESTKR" >$OUT
+
+if [ -s "$OUT" ]; then
+  /usr/sbin/sendmail -odq -oi -t <<EOF
+From: $ID
+To: <debian-project@lists.debian.org>
+Subject: Updated Debian Maintainers Keyring
+Content-Type: text/plain; charset=utf-8
+MIME-Version: 1.0
+
+With the upload of debian-maintainers version $VERSION, the following
+changes to the keyring have been made:
+
+$(cat $OUT)
+
+A summary of all the changes in this upload follows.
+
+Debian distribution maintenance software,
+on behalf of,
+$ID
+
+$(cat $CHANGES)
+EOF
+fi
+rm -f "$OUT"
+
+exit 0

scripts/debian/dm-monitor

+#!/bin/sh
+
+echo "Known debian maintainers:"
+
+psql --html projectb <<EOF
+  SELECT uid.uid, uid.name, f.fingerprint
+    FROM uid LEFT OUTER JOIN fingerprint f ON (uid.id = f.uid) 
+   WHERE uid.uid LIKE 'dm:%'
+ORDER BY uid.uid;
+EOF
+
+echo "Packages debian maintainers may update:"
+
+psql --html projectb <<EOF
+  SELECT s.source, s.version, u.uid
+    FROM src_uploaders su JOIN source s ON (su.source = s.id) 
+         JOIN src_associations sa ON (s.id = sa.source)
+         JOIN maintainer m ON (su.maintainer = m.id)
+         JOIN uid u ON (m.name LIKE u.name || ' <%>')
+   WHERE u.uid LIKE 'dm:%' AND sa.suite = 5
+ORDER BY u.uid, s.source, s.version;
+EOF
+
+echo "Source packages in the pool uploaded by debian maintainers:"
+
+psql --html projectb <<EOF
+  SELECT s.source, s.version, s.install_date, u.uid
+    FROM source s JOIN fingerprint f ON (s.sig_fpr = f.id) 
+         JOIN uid u ON (f.uid = u.id)
+   WHERE u.uid LIKE 'dm:%'
+ORDER BY u.uid, s.source, s.version;
+EOF
+
+echo "Binary packages in the pool uploaded by debian maintainers:"
+
+psql --html projectb <<EOF
+  SELECT b.package, b.version, a.arch_string AS arch, u.uid
+    FROM binaries b JOIN architecture a ON (b.architecture = a.id)
+         JOIN fingerprint f ON (b.sig_fpr = f.id) 
+         JOIN uid u ON (f.uid = u.id)
+   WHERE u.uid LIKE 'dm:%'
+ORDER BY u.uid, b.package, b.version;
+EOF
+
+echo "Recorded Uploaders:"
+
+psql --html projectb <<EOF
+  SELECT s.source, s.version, m.name
+    FROM src_uploaders su JOIN source s ON (su.source = s.id) 
+         JOIN maintainer m ON (su.maintainer = m.id)
+ORDER BY m.name, s.source, s.version;
+EOF
+
+echo "Keys without a recorded uid:"
+
+psql --html projectb <<EOF
+  SELECT *
+    FROM fingerprint f
+   WHERE f.uid IS NULL;
+EOF
+