Merge commit 'ncomm/dm_upload_allowed' into merge

* commit 'ncomm/dm_upload_allowed':
  Fixed to handle multiple DD/DM keyrings, and ldap to handle new DM status
  Modified dm status to be a keyring specific switch, modified update1
  Fixed it so the fingerprint table is properly referenced vs the uid
  fix typo
  Fixed and made changes for Joerg.
  Actually use the sleep call ...
  Updated update_db.py to use a lockfile properly and to properly wait.
  fixed update_db not to have a braindead mistake. Don't write code at 1 in the
  Added update system for the database, and the first update script
  Changed process_unchecked.py to handle DM's (slightly) more sanely.
  Modified dak to use non-braindead DM schema, and use an actual column for
  Corrected a bug with the handling of DM-Upload-Allowed and experimental suites of the archive
  Added dm-upload-allowed flag to exist, and added support for all
Signed-off-by: NJoerg Jaspert <joerg@debian.org>

config/debian/dak.conf

@@ -126,6 +126,14 @@ Import-Archive
   ExportDir "/srv/ftp.debian.org/dak/import-archive-files/";
 };
 
+Import-Keyring
+{
+  /srv/ftp.debian.org/keyrings/debian-maintainers.gpg
+    {
+      Debian-Maintainer "true";
+    };
+};
+
 Reject-Proposed-Updates
 {
    StableRejector "the Stable Release Team";

dak/dak.py

@@ -144,6 +144,8 @@ def init():
          "Sync PostgreSQL users with passwd file"),
         ("init-db",
          "Update the database to match the conf file"),
+        ("update-db",
+         "Updates databae schema to latest revision"),
         ("init-dirs",
          "Initial setup of the archive"),
         ("make-maintainers",

dak/dakdb/update1.py

+#!/usr/bin/env python
+
+# Debian Archive Kit Database Update Script
+# Copyright (C) 2008  Michael Casadevall <mcasadevall@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+################################################################################
+
+# <tomv_w> really, if we want to screw ourselves, let's find a better way.
+# <Ganneff> rm -rf /srv/ftp.debian.org
+
+################################################################################
+
+import psycopg2, time
+
+################################################################################
+
+def do_update(self):
+    print "Adding DM fields to database"
+
+    try:
+        c = self.db.cursor()
+        c.execute("ALTER TABLE source ADD COLUMN dm_upload_allowed BOOLEAN DEFAULT 'no' NOT NULL;")
+        c.execute("ALTER TABLE keyrings ADD COLUMN debian_maintainer BOOLEAN DEFAULT 'false' NOT NULL;")
+
+        print "Migrating DM data to source table. This might take some time ..."
+
+        c.execute("UPDATE source SET dm_upload_allowed = 't' WHERE id IN (SELECT source FROM src_uploaders);")
+        c.execute("UPDATE config SET value = '1' WHERE name = 'db_revision'")
+
+        print "Migrating DM uids to normal uids"
+        c.execute("SELECT uid FROM uid WHERE uid  LIKE 'dm:%'")
+        rows = c.fetchall()
+        for r in rows:
+            uid = r[0]
+            c.execute("UPDATE uid SET uid = '%s' WHERE uid = '%s'" % (uid[3:], uid))
+
+        self.db.commit()
+
+        print "IMPORTANT: Set the debian_maintainer flag in the config file for keyrings that are DMs!"
+        print "           Failure to do so will result in DM's having full upload abilities!"
+        print "REMINDER: Remember to run the updated byhand-dm crontab to update Debian Maintainer information"
+        print ""
+        print "Pausing for five seconds ..."
+        time.sleep (5)
+
+    except psycopg2.ProgrammingError, msg:
+        self.db.rollback()
+        print "FATAL: Unable to apply DM table update 1!"
+        print "Error Message: " + str(msg)
+        print "Database changes have been rolled back."

dak/import_keyring.py

@@ -222,6 +222,11 @@ def main():
     keyringname = keyring_names[0]
     keyring = Keyring(keyringname)
 
+    is_dm = "false"
+    if Cnf.has_key("Import-Keyring::"+keyringname+"::Debian-Maintainer"):
+        projectB.query("UPDATE keyrings SET debian_maintainer = '%s' WHERE name = '%s'" % (Cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"], keyringname.split("/")[-1]))
+        is_dm = Cnf["Import-Keyring::"+keyringname+"::Debian-Maintainer"]
+
     keyring_id = database.get_or_set_keyring_id(
                         keyringname.split("/")[-1])
 
@@ -291,7 +296,12 @@ def main():
                 projectB.query("UPDATE fingerprint SET uid = %d WHERE id = %d" % (newuid, oldfid))
 
             if oldkid != keyring_id:
-                projectB.query("UPDATE fingerprint SET keyring = %d WHERE id = %d" % (keyring_id, oldfid))
+                # Only change the keyring if it won't result in a loss of permissions
+                q = projectB.query("SELECT debian_maintainer FROM keyrings WHERE id = '%d'" % (keyring_id))
+                if is_dm == "false" and q.getresult()[0][0] == 'f':
+                    projectB.query("UPDATE fingerprint SET keyring = %d WHERE id = %d" % (keyring_id, oldfid))
+                else:
+                    print "Key %s exists in both DM and DD keyrings. Not demoting." % (f)
 
     # All done!
 

dak/import_ldap_fingerprints.py

@@ -151,7 +151,7 @@ SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id
                     print "Assigning %s to 0x%s." % (uid, fingerprint)
                 elif existing_uid == uid:
                     pass
-                elif existing_uid[:3] == "dm:":
+                elif '@' not in existing_ui:
                     q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id))
                     print "Promoting DM %s to DD %s with keyid 0x%s." % (existing_uid, uid, fingerprint)
                 else:

dak/process_accepted.py

@@ -298,10 +298,14 @@ def install ():
             filename = files[file]["pool name"] + file
             dsc_component = files[file]["component"]
             dsc_location_id = files[file]["location id"]
+            if dsc.has_key("dm-upload-allowed") and  dsc["dm-upload-allowed"] == "yes":
+                dm_upload_allowed = "true"
+            else:
+                dm_upload_allowed = "false"
             if not files[file].has_key("files id") or not files[file]["files id"]:
                 files[file]["files id"] = database.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["sha1sum"], files[file]["sha256sum"], dsc_location_id)
-            projectB.query("INSERT INTO source (source, version, maintainer, changedby, file, install_date, sig_fpr) VALUES ('%s', '%s', %d, %d, %d, '%s', %s)"
-                           % (package, version, maintainer_id, changedby_id, files[file]["files id"], install_date, fingerprint_id))
+            projectB.query("INSERT INTO source (source, version, maintainer, changedby, file, install_date, sig_fpr, dm_upload_allowed) VALUES ('%s', '%s', %d, %d, %d, '%s', %s, %s)"
+                           % (package, version, maintainer_id, changedby_id, files[file]["files id"], install_date, fingerprint_id, dm_upload_allowed))
 
             for suite in changes["distribution"].keys():
                 suite_id = database.get_suite_id(suite)
@@ -322,21 +326,20 @@ def install ():
                 projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id))
 
             # Add the src_uploaders to the DB
-            if dsc.get("dm-upload-allowed", "no") == "yes":
-                uploader_ids = [maintainer_id]
-                if dsc.has_key("uploaders"):
-                    for u in dsc["uploaders"].split(","):
-                        u = u.replace("'", "\\'")
-                        u = u.strip()
-                        uploader_ids.append(
-                            database.get_or_set_maintainer_id(u))
-                added_ids = {}
-                for u in uploader_ids:
-                    if added_ids.has_key(u):
-                        utils.warn("Already saw uploader %s for source %s" % (u, package))
-                        continue
-                    added_ids[u]=1
-                    projectB.query("INSERT INTO src_uploaders (source, maintainer) VALUES (currval('source_id_seq'), %d)" % (u))
+            uploader_ids = [maintainer_id]
+            if dsc.has_key("uploaders"):
+                for u in dsc["uploaders"].split(","):
+                    u = u.replace("'", "\\'")
+                    u = u.strip()
+                    uploader_ids.append(
+                        database.get_or_set_maintainer_id(u))
+            added_ids = {}
+            for u in uploader_ids:
+                if added_ids.has_key(u):
+                    utils.warn("Already saw uploader %s for source %s" % (u, package))
+                    continue
+                added_ids[u]=1
+                projectB.query("INSERT INTO src_uploaders (source, maintainer) VALUES (currval('source_id_seq'), %d)" % (u))
 
 
     # Add the .deb files to the DB

dak/process_unchecked.py

@@ -997,7 +997,7 @@ def check_timestamps():
 ################################################################################
 
 def lookup_uid_from_fingerprint(fpr):
-    q = Upload.projectB.query("SELECT u.uid, u.name FROM fingerprint f, uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr))
+    q = Upload.projectB.query("SELECT u.uid, u.name, k.debian_maintainer FROM fingerprint f JOIN keyrings k ON (f.keyring=k.id), uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr))
     qs = q.getresult()
     if len(qs) == 0:
         return (None, None)
@@ -1007,7 +1007,7 @@ def lookup_uid_from_fingerprint(fpr):
 def check_signed_by_key():
     """Ensure the .changes is signed by an authorized uploader."""
 
-    (uid, uid_name) = lookup_uid_from_fingerprint(changes["fingerprint"])
+    (uid, uid_name, is_dm) = lookup_uid_from_fingerprint(changes["fingerprint"])
     if uid_name == None:
         uid_name = ""
 
@@ -1017,8 +1017,8 @@ def check_signed_by_key():
         may_nmu, may_sponsor = 1, 1
         # XXX by default new dds don't have a fingerprint/uid in the db atm,
         #     and can't get one in there if we don't allow nmu/sponsorship
-    elif uid[:3] == "dm:":
-        uid_email = uid[3:]
+    elif is_dm is "t":
+        uid_email = uid
         may_nmu, may_sponsor = 0, 0
     else:
         uid_email = "%s@debian.org" % (uid)
@@ -1043,25 +1043,28 @@ def check_signed_by_key():
 
     if not sponsored and not may_nmu:
         source_ids = []
-        check_suites = changes["distribution"].keys()
-        if "unstable" not in check_suites: check_suites.append("unstable")
-        for suite in check_suites:
-            suite_id = database.get_suite_id(suite)
-            q = Upload.projectB.query("SELECT s.id FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND sa.suite = %d" % (changes["source"], suite_id))
-            for si in q.getresult():
-                if si[0] not in source_ids: source_ids.append(si[0])
-
-        is_nmu = 1
-        for si in source_ids:
-            is_nmu = 1
-            q = Upload.projectB.query("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT maintainer FROM src_uploaders WHERE src_uploaders.source = %s)" % (si))
+        q = Upload.projectB.query("SELECT s.id, s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND s.dm_upload_allowed = 'yes'" % (changes["source"]))
+
+        highest_sid, highest_version = None, None
+
+        should_reject = True
+        for si in q.getresult():
+            if highest_version == None or apt_pkg.VersionCompare(si[1], highest_version) == 1:
+                 highest_sid = si[0]
+                 highest_version = si[1]
+
+        if highest_sid == None:
+           reject("Source package %s does not have 'DM-Upload-Allowed: yes' in its most recent version" % changes["source"])
+        else:
+            q = Upload.projectB.query("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT su.maintainer FROM src_uploaders su JOIN source s ON (s.id = su.source) WHERE su.source = %s)" % (highest_sid))
             for m in q.getresult():
                 (rfc822, rfc2047, name, email) = utils.fix_maintainer(m[0])
                 if email == uid_email or name == uid_name:
-                    is_nmu=0
+                    should_reject=True
                     break
-        if is_nmu:
-            reject("%s may not upload/NMU source package %s" % (uid, changes["source"]))
+
+        if should_reject == True:
+            reject("%s is not in Maintainer or Uploaders of source package %s" % (uid, changes["source"]))
 
         for b in changes["binary"].keys():
             for suite in changes["distribution"].keys():

dak/update_db.py

+#!/usr/bin/env python
+
+# Debian Archive Kit Database Update Script
+# Copyright (C) 2008  Michael Casadevall <mcasadevall@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+################################################################################
+
+# <Ganneff> when do you have it written?
+# <NCommander> Ganneff, after you make my debian account
+# <Ganneff> blackmail wont work
+# <NCommander> damn it
+
+################################################################################
+
+import psycopg2, sys, fcntl, os
+import apt_pkg
+import time
+from daklib import database
+from daklib import utils
+
+################################################################################
+
+Cnf = None
+projectB = None
+required_database_schema = 1
+
+################################################################################
+
+class UpdateDB:
+    def usage (self, exit_code=0):
+        print """Usage: dak update-db
+Updates dak's database schema to the lastest version. You should disable crontabs while this is running
+
+  -h, --help                show this help and exit."""
+        sys.exit(exit_code)
+
+
+################################################################################
+
+    def update_db_to_zero(self):
+        # This function will attempt to update a pre-zero database schema to zero
+
+        # First, do the sure thing, and create the configuration table
+        try:
+            print "Creating configuration table ..."
+            c = self.db.cursor()
+            c.execute("""CREATE TABLE config (
+                                  id SERIAL PRIMARY KEY NOT NULL,
+                                  name TEXT UNIQUE NOT NULL,
+                                  value TEXT
+                                );""")
+            c.execute("INSERT INTO config VALUES ( nextval('config_id_seq'), 'db_revision', '0')");
+            self.db.commit()
+
+        except psycopg2.ProgrammingError:
+            self.db.rollback()
+            print "Failed to create configuration table."
+            print "Can the projectB user CREATE TABLE?"
+            print ""
+            print "Aborting update."
+            sys.exit(-255)
+
+################################################################################
+
+    def get_db_rev(self):
+        global projectB
+
+        # We keep database revision info the config table
+        # Try and access it
+
+        try:
+            c = self.db.cursor()
+            q = c.execute("SELECT value FROM config WHERE name = 'db_revision';");
+            return c.fetchone()[0]
+
+        except psycopg2.ProgrammingError:
+            # Whoops .. no config table ...
+            self.db.rollback()
+            print "No configuration table found, assuming dak database revision to be pre-zero"
+            return -1
+
+################################################################################
+
+    def update_db(self):
+        # Ok, try and find the configuration table
+        print "Determining dak database revision ..."
+
+        try:
+            self.db = psycopg2.connect("dbname='" + Cnf["DB::Name"] + "' host='" + Cnf["DB::Host"] + "' port='" + str(Cnf["DB::Port"]) + "'")
+
+        except:
+            print "FATAL: Failed connect to database"
+            pass
+
+        database_revision = int(self.get_db_rev())
+
+        if database_revision == -1:
+            print "dak database schema predates update-db."
+            print ""
+            print "This script will attempt to upgrade it to the lastest, but may fail."
+            print "Please make sure you have a database backup handy. If you don't, press Ctrl-C now!"
+            print ""
+            print "Continuing in five seconds ..."
+            time.sleep(5)
+            print ""
+            print "Attempting to upgrade pre-zero database to zero"
+
+            self.update_db_to_zero()
+            database_revision = 0
+
+        print "dak database schema at " + str(database_revision)
+        print "dak version requires schema " + str(required_database_schema)
+
+        if database_revision == required_database_schema:
+            print "no updates required"
+            sys.exit(0)
+
+        for i in range (database_revision, required_database_schema):
+            print "updating databse schema from " + str(database_revision) + " to " + str(i+1)
+            dakdb = __import__("dakdb", globals(), locals(), ['update'+str(i+1)])
+            update_module = getattr(dakdb, "update"+str(i+1))
+            update_module.do_update(self)
+            database_revision += 1
+
+################################################################################
+
+    def init (self):
+        global Cnf, projectB
+
+        Cnf = utils.get_conf()
+        arguments = [('h', "help", "Update-DB::Options::Help")]
+        for i in [ "help" ]:
+            if not Cnf.has_key("Update-DB::Options::%s" % (i)):
+                Cnf["Update-DB::Options::%s" % (i)] = ""
+
+        arguments = apt_pkg.ParseCommandLine(Cnf, arguments, sys.argv)
+
+        options = Cnf.SubTree("Update-DB::Options")
+        if options["Help"]:
+            usage()
+        elif arguments:
+            utils.warn("dak update-db takes no arguments.")
+            usage(exit_code=1)
+
+
+        self.update_db()
+
+        try:
+            lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT)
+            fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError, e:
+            if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN':
+                utils.fubar("Couldn't obtain lock; assuming another 'dak process-unchecked' is already running.")
+
+
+################################################################################
+
+if __name__ == '__main__':
+    app = UpdateDB()
+    app.init()
+
+def main():
+    app = UpdateDB()
+    app.init()

scripts/debian/byhand-dm

@@ -41,7 +41,7 @@ echo "Authorised upload by $ID, copying into place"
 OUT=$(mktemp)
 
 cp "$BYHAND" "$DESTKR"
-dak import-keyring --generate-users "dm:%s" "$DESTKR" >$OUT
+dak import-keyring -D --generate-users "%s" "$DESTKR" >$OUT
 
 if [ -s "$OUT" ]; then
   /usr/sbin/sendmail -odq -oi -t <<EOF