archive.py

# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

"""module to manipulate the archive

This module provides classes to manipulate the archive.
"""

from daklib.dbconn import *
import daklib.checks as checks
from daklib.config import Config
import daklib.upload as upload
import daklib.utils as utils
from daklib.fstransactions import FilesystemTransaction
from daklib.regexes import re_changelog_versions, re_bin_only_nmu
import daklib.daksubprocess

import apt_pkg
from datetime import datetime
import os
import shutil
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.orm import object_session
import sqlalchemy.exc
import tempfile
import traceback

class ArchiveException(Exception):
    pass

class HashMismatchException(ArchiveException):
    pass

class ArchiveTransaction(object):
    """manipulate the archive in a transaction
    """
    def __init__(self):
        self.fs = FilesystemTransaction()
        self.session = DBConn().session()

    def get_file(self, hashed_file, source_name, check_hashes=True):
        """Look for file C{hashed_file} in database

        @type  hashed_file: L{daklib.upload.HashedFile}
        @param hashed_file: file to look for in the database

        @type  source_name: str
        @param source_name: source package name

        @type  check_hashes: bool
        @param check_hashes: check size and hashes match

        @raise KeyError: file was not found in the database
        @raise HashMismatchException: hash mismatch

        @rtype:  L{daklib.dbconn.PoolFile}
        @return: database entry for the file
        """
        poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
        try:
            poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
            if check_hashes and (poolfile.filesize != hashed_file.size
                                 or poolfile.md5sum != hashed_file.md5sum
                                 or poolfile.sha1sum != hashed_file.sha1sum
                                 or poolfile.sha256sum != hashed_file.sha256sum):
                raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
            return poolfile
        except NoResultFound:
            raise KeyError('{0} not found in database.'.format(poolname))

    def _install_file(self, directory, hashed_file, archive, component, source_name):
        """Install a file

        Will not give an error when the file is already present.

        @rtype:  L{daklib.dbconn.PoolFile}
        @return: database object for the new file
        """
        session = self.session

        poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
        try:
            poolfile = self.get_file(hashed_file, source_name)
        except KeyError:
            poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
            poolfile.md5sum = hashed_file.md5sum
            poolfile.sha1sum = hashed_file.sha1sum
            poolfile.sha256sum = hashed_file.sha256sum
            session.add(poolfile)
            session.flush()

        try:
            session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
        except NoResultFound:
            archive_file = ArchiveFile(archive, component, poolfile)
            session.add(archive_file)
            session.flush()

            path = os.path.join(archive.path, 'pool', component.component_name, poolname)
            hashed_file_path = os.path.join(directory, hashed_file.input_filename)
            self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)

        return poolfile

    def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
        """Install a binary package

        @type  directory: str
        @param directory: directory the binary package is located in

        @type  binary: L{daklib.upload.Binary}
        @param binary: binary package to install

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: target suite

        @type  component: L{daklib.dbconn.Component}
        @param component: target component

        @type  allow_tainted: bool
        @param allow_tainted: allow to copy additional files from tainted archives

        @type  fingerprint: L{daklib.dbconn.Fingerprint}
        @param fingerprint: optional fingerprint

        @type  source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
        @param source_suites: suites to copy the source from if they are not
                              in C{suite} or C{True} to allow copying from any
                              suite.

        @type  extra_source_archives: list of L{daklib.dbconn.Archive}
        @param extra_source_archives: extra archives to copy Built-Using sources from

        @rtype:  L{daklib.dbconn.DBBinary}
        @return: databse object for the new package
        """
        session = self.session
        control = binary.control
        maintainer = get_or_set_maintainer(control['Maintainer'], session)
        architecture = get_architecture(control['Architecture'], session)

        (source_name, source_version) = binary.source
        source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
        source = source_query.filter(DBSource.suites.contains(suite)).first()
        if source is None:
            if source_suites != True:
                source_query = source_query.join(DBSource.suites) \
                    .filter(Suite.suite_id == source_suites.c.id)
            source = source_query.first()
            if source is None:
                raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
            self.copy_source(source, suite, component)

        db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)

        unique = dict(
            package=control['Package'],
            version=control['Version'],
            architecture=architecture,
            )
        rest = dict(
            source=source,
            maintainer=maintainer,
            poolfile=db_file,
            binarytype=binary.type,
            )
        # Other attributes that are ignored for purposes of equality with
        # an existing source
        rest2 = dict(
            fingerprint=fingerprint,
            )

        try:
            db_binary = session.query(DBBinary).filter_by(**unique).one()
            for key, value in rest.iteritems():
                if getattr(db_binary, key) != value:
                    raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
        except NoResultFound:
            db_binary = DBBinary(**unique)
            for key, value in rest.iteritems():
                setattr(db_binary, key, value)
            for key, value in rest2.iteritems():
                setattr(db_binary, key, value)
            session.add(db_binary)
            session.flush()
            import_metadata_into_db(db_binary, session)

            self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)

        if suite not in db_binary.suites:
            db_binary.suites.append(suite)

        session.flush()

        return db_binary

    def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
        """ensure source exists in the given archive

        This is intended to be used to check that Built-Using sources exist.

        @type  filename: str
        @param filename: filename to use in error messages

        @type  source: L{daklib.dbconn.DBSource}
        @param source: source to look for

        @type  archive: L{daklib.dbconn.Archive}
        @param archive: archive to look in

        @type  extra_archives: list of L{daklib.dbconn.Archive}
        @param extra_archives: list of archives to copy the source package from
                               if it is not yet present in C{archive}
        """
        session = self.session
        db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
        if db_file is not None:
            return True

        # Try to copy file from one extra archive
        if extra_archives is None:
            extra_archives = []
        db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
        if db_file is None:
            raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))

        source_archive = db_file.archive
        for dsc_file in source.srcfiles:
            af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
            # We were given an explicit list of archives so it is okay to copy from tainted archives.
            self._copy_file(af.file, archive, db_file.component, allow_tainted=True)

    def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
        """Add Built-Using sources to C{db_binary.extra_sources}
        """
        session = self.session

        for bu_source_name, bu_source_version in daklib.utils.parse_built_using(control):
            bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
            if bu_source is None:
                raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))

            self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)

            db_binary.extra_sources.append(bu_source)

    def install_source_to_archive(self, directory, source, archive, component, changed_by, allow_tainted=False, fingerprint=None):
        session = self.session
        control = source.dsc
        maintainer = get_or_set_maintainer(control['Maintainer'], session)
        source_name = control['Source']

        ### Add source package to database

        # We need to install the .dsc first as the DBSource object refers to it.
        db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)

        unique = dict(
            source=source_name,
            version=control['Version'],
            )
        rest = dict(
            maintainer=maintainer,
            poolfile=db_file_dsc,
            dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
            )
        # Other attributes that are ignored for purposes of equality with
        # an existing source
        rest2 = dict(
            changedby=changed_by,
            fingerprint=fingerprint,
            )

        created = False
        try:
            db_source = session.query(DBSource).filter_by(**unique).one()
            for key, value in rest.iteritems():
                if getattr(db_source, key) != value:
                    raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
        except NoResultFound:
            created = True
            db_source = DBSource(**unique)
            for key, value in rest.iteritems():
                setattr(db_source, key, value)
            for key, value in rest2.iteritems():
                setattr(db_source, key, value)
            session.add(db_source)
            session.flush()

            # Add .dsc file. Other files will be added later.
            db_dsc_file = DSCFile()
            db_dsc_file.source = db_source
            db_dsc_file.poolfile = db_file_dsc
            session.add(db_dsc_file)
            session.flush()

        if not created:
            for f in db_source.srcfiles:
                self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
            return db_source

        ### Now add remaining files and copy them to the archive.

        for hashed_file in source.files.itervalues():
            hashed_file_path = os.path.join(directory, hashed_file.input_filename)
            if os.path.exists(hashed_file_path):
                db_file = self._install_file(directory, hashed_file, archive, component, source_name)
                session.add(db_file)
            else:
                db_file = self.get_file(hashed_file, source_name)
                self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)

            db_dsc_file = DSCFile()
            db_dsc_file.source = db_source
            db_dsc_file.poolfile = db_file
            session.add(db_dsc_file)

        session.flush()

        # Importing is safe as we only arrive here when we did not find the source already installed earlier.
        import_metadata_into_db(db_source, session)

        # Uploaders are the maintainer and co-maintainers from the Uploaders field
        db_source.uploaders.append(maintainer)
        if 'Uploaders' in control:
            from daklib.textutils import split_uploaders
            for u in split_uploaders(control['Uploaders']):
                db_source.uploaders.append(get_or_set_maintainer(u, session))
        session.flush()

        return db_source

    def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
        """Install a source package

        @type  directory: str
        @param directory: directory the source package is located in

        @type  source: L{daklib.upload.Source}
        @param source: source package to install

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: target suite

        @type  component: L{daklib.dbconn.Component}
        @param component: target component

        @type  changed_by: L{daklib.dbconn.Maintainer}
        @param changed_by: person who prepared this version of the package

        @type  allow_tainted: bool
        @param allow_tainted: allow to copy additional files from tainted archives

        @type  fingerprint: L{daklib.dbconn.Fingerprint}
        @param fingerprint: optional fingerprint

        @rtype:  L{daklib.dbconn.DBSource}
        @return: database object for the new source
        """
        db_source = self.install_source_to_archive(directory, source, suite.archive, component, changed_by, allow_tainted, fingerprint)

        if suite in db_source.suites:
            return db_source
        db_source.suites.append(suite)
        self.session.flush()

        return db_source

    def _copy_file(self, db_file, archive, component, allow_tainted=False):
        """Copy a file to the given archive and component

        @type  db_file: L{daklib.dbconn.PoolFile}
        @param db_file: file to copy

        @type  archive: L{daklib.dbconn.Archive}
        @param archive: target archive

        @type  component: L{daklib.dbconn.Archive}
        @param component: target component

        @type  allow_tainted: bool
        @param allow_tainted: allow to copy from tainted archives (such as NEW)
        """
        session = self.session

        if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
            query = session.query(ArchiveFile).filter_by(file=db_file)
            if not allow_tainted:
                query = query.join(Archive).filter(Archive.tainted == False)

            source_af = query.first()
            if source_af is None:
                raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename))
            target_af = ArchiveFile(archive, component, db_file)
            session.add(target_af)
            session.flush()
            self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)

    def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
        """Copy a binary package to the given suite and component

        @type  db_binary: L{daklib.dbconn.DBBinary}
        @param db_binary: binary to copy

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: target suite

        @type  component: L{daklib.dbconn.Component}
        @param component: target component

        @type  allow_tainted: bool
        @param allow_tainted: allow to copy from tainted archives (such as NEW)

        @type  extra_archives: list of L{daklib.dbconn.Archive}
        @param extra_archives: extra archives to copy Built-Using sources from
        """
        session = self.session
        archive = suite.archive
        if archive.tainted:
            allow_tainted = True

        filename = db_binary.poolfile.filename

        # make sure source is present in target archive
        db_source = db_binary.source
        if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
            raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))

        # make sure built-using packages are present in target archive
        for db_source in db_binary.extra_sources:
            self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)

        # copy binary
        db_file = db_binary.poolfile
        self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
        if suite not in db_binary.suites:
            db_binary.suites.append(suite)
        self.session.flush()

    def copy_source(self, db_source, suite, component, allow_tainted=False):
        """Copy a source package to the given suite and component

        @type  db_source: L{daklib.dbconn.DBSource}
        @param db_source: source to copy

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: target suite

        @type  component: L{daklib.dbconn.Component}
        @param component: target component

        @type  allow_tainted: bool
        @param allow_tainted: allow to copy from tainted archives (such as NEW)
        """
        archive = suite.archive
        if archive.tainted:
            allow_tainted = True
        for db_dsc_file in db_source.srcfiles:
            self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
        if suite not in db_source.suites:
            db_source.suites.append(suite)
        self.session.flush()

    def remove_file(self, db_file, archive, component):
        """Remove a file from a given archive and component

        @type  db_file: L{daklib.dbconn.PoolFile}
        @param db_file: file to remove

        @type  archive: L{daklib.dbconn.Archive}
        @param archive: archive to remove the file from

        @type  component: L{daklib.dbconn.Component}
        @param component: component to remove the file from
        """
        af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
        self.fs.unlink(af.path)
        self.session.delete(af)

    def remove_binary(self, binary, suite):
        """Remove a binary from a given suite and component

        @type  binary: L{daklib.dbconn.DBBinary}
        @param binary: binary to remove

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: suite to remove the package from
        """
        binary.suites.remove(suite)
        self.session.flush()

    def remove_source(self, source, suite):
        """Remove a source from a given suite and component

        @type  source: L{daklib.dbconn.DBSource}
        @param source: source to remove

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: suite to remove the package from

        @raise ArchiveException: source package is still referenced by other
                                 binaries in the suite
        """
        session = self.session

        query = session.query(DBBinary).filter_by(source=source) \
            .filter(DBBinary.suites.contains(suite))
        if query.first() is not None:
            raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))

        source.suites.remove(suite)
        session.flush()

    def commit(self):
        """commit changes"""
        try:
            self.session.commit()
            self.fs.commit()
        finally:
            self.session.rollback()
            self.fs.rollback()

    def rollback(self):
        """rollback changes"""
        self.session.rollback()
        self.fs.rollback()

    def flush(self):
        self.session.flush()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        if type is None:
            self.commit()
        else:
            self.rollback()
        return None

def source_component_from_package_list(package_list, suite):
    """Get component for a source package

    This function will look at the Package-List field to determine the
    component the source package belongs to. This is the first component
    the source package provides binaries for (first with respect to the
    ordering of components).

    It the source package has no Package-List field, None is returned.

    @type  package_list: L{daklib.packagelist.PackageList}
    @param package_list: package list of the source to get the override for

    @type  suite: L{daklib.dbconn.Suite}
    @param suite: suite to consider for binaries produced

    @rtype:  L{daklib.dbconn.Component} or C{None}
    @return: component for the given source or C{None}
    """
    if package_list.fallback:
        return None
    session = object_session(suite)
    packages = package_list.packages_for_suite(suite)
    components = set(p.component for p in packages)
    query = session.query(Component).order_by(Component.ordering) \
            .filter(Component.component_name.in_(components))
    return query.first()

class ArchiveUpload(object):
    """handle an upload

    This class can be used in a with-statement::

       with ArchiveUpload(...) as upload:
          ...

    Doing so will automatically run any required cleanup and also rollback the
    transaction if it was not committed.
    """
    def __init__(self, directory, changes, keyrings):
        self.transaction = ArchiveTransaction()
        """transaction used to handle the upload
        @type: L{daklib.archive.ArchiveTransaction}
        """

        self.session = self.transaction.session
        """database session"""

        self.original_directory = directory
        self.original_changes = changes

        self.changes = None
        """upload to process
        @type: L{daklib.upload.Changes}
        """

        self.directory = None
        """directory with temporary copy of files. set by C{prepare}
        @type: str
        """

        self.keyrings = keyrings

        self.fingerprint = self.session.query(Fingerprint).filter_by(fingerprint=changes.primary_fingerprint).one()
        """fingerprint of the key used to sign the upload
        @type: L{daklib.dbconn.Fingerprint}
        """

        self.reject_reasons = []
        """reasons why the upload cannot by accepted
        @type: list of str
        """

        self.warnings = []
        """warnings
        @note: Not used yet.
        @type: list of str
        """

        self.final_suites = None

        self.new = False
        """upload is NEW. set by C{check}
        @type: bool
        """

        self._checked = False
        """checks passes. set by C{check}
        @type: bool
        """

        self._new_queue = self.session.query(PolicyQueue).filter_by(queue_name='new').one()
        self._new = self._new_queue.suite

    def warn(self, message):
        """add a warning message

        Adds a warning message that can later be seen in C{self.warnings}

        @type  message: string
        @param message: warning message
        """
        self.warnings.append(message)

    def prepare(self):
        """prepare upload for further processing

        This copies the files involved to a temporary directory.  If you use
        this method directly, you have to remove the directory given by the
        C{directory} attribute later on your own.

        Instead of using the method directly, you can also use a with-statement::

           with ArchiveUpload(...) as upload:
              ...

        This will automatically handle any required cleanup.
        """
        assert self.directory is None
        assert self.original_changes.valid_signature

        cnf = Config()
        session = self.transaction.session

        group = cnf.get('Dinstall::UnprivGroup') or None
        self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'),
                                            mode=0o2750, group=group)
        with FilesystemTransaction() as fs:
            src = os.path.join(self.original_directory, self.original_changes.filename)
            dst = os.path.join(self.directory, self.original_changes.filename)
            fs.copy(src, dst, mode=0o640)

            self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings)

            for f in self.changes.files.itervalues():
                src = os.path.join(self.original_directory, f.filename)
                dst = os.path.join(self.directory, f.filename)
                if not os.path.exists(src):
                    continue
                fs.copy(src, dst, mode=0o640)

            source = None
            try:
                source = self.changes.source
            except Exception:
                # Do not raise an exception here if the .dsc is invalid.
                pass

            if source is not None:
                for f in source.files.itervalues():
                    src = os.path.join(self.original_directory, f.filename)
                    dst = os.path.join(self.directory, f.filename)
                    if not os.path.exists(dst):
                        try:
                            db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False)
                            db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first()
                            fs.copy(db_archive_file.path, dst, mode=0o640)
                        except KeyError:
                            # Ignore if get_file could not find it. Upload will
                            # probably be rejected later.
                            pass

    def unpacked_source(self):
        """Path to unpacked source

        Get path to the unpacked source. This method does unpack the source
        into a temporary directory under C{self.directory} if it has not
        been done so already.

        @rtype:  str or C{None}
        @return: string giving the path to the unpacked source directory
                 or C{None} if no source was included in the upload.
        """
        assert self.directory is not None

        source = self.changes.source
        if source is None:
            return None
        dsc_path = os.path.join(self.directory, source._dsc_file.filename)

        sourcedir = os.path.join(self.directory, 'source')
        if not os.path.exists(sourcedir):
            devnull = open('/dev/null', 'w')
            daklib.daksubprocess.check_call(["dpkg-source", "--no-copy", "--no-check", "-x", dsc_path, sourcedir], shell=False, stdout=devnull)
        if not os.path.isdir(sourcedir):
            raise Exception("{0} is not a directory after extracting source package".format(sourcedir))
        return sourcedir

    def _map_suite(self, suite_name):
        for rule in Config().value_list("SuiteMappings"):
            fields = rule.split()
            rtype = fields[0]
            if rtype == "map" or rtype == "silent-map":
                (src, dst) = fields[1:3]
                if src == suite_name:
                    suite_name = dst
                    if rtype != "silent-map":
                        self.warnings.append('Mapping {0} to {1}.'.format(src, dst))
            elif rtype == "ignore":
                ignored = fields[1]
                if suite_name == ignored:
                    self.warnings.append('Ignoring target suite {0}.'.format(ignored))
                    suite_name = None
            elif rtype == "reject":
                rejected = fields[1]
                if suite_name == rejected:
                    raise checks.Reject('Uploads to {0} are not accepted.'.format(rejected))
            ## XXX: propup-version and map-unreleased not yet implemented
        return suite_name

    def _mapped_suites(self):
        """Get target suites after mappings

        @rtype:  list of L{daklib.dbconn.Suite}
        @return: list giving the mapped target suites of this upload
        """
        session = self.session

        suite_names = []
        for dist in self.changes.distributions:
            suite_name = self._map_suite(dist)
            if suite_name is not None:
                suite_names.append(suite_name)

        suites = session.query(Suite).filter(Suite.suite_name.in_(suite_names))
        return suites

    def _check_new_binary_overrides(self, suite, overridesuite):
        new = False
        source = self.changes.source

        # Check binaries listed in the source package's Package-List field:
        if source is not None and not source.package_list.fallback:
            packages = source.package_list.packages_for_suite(suite)
            binaries = [ entry for entry in packages ]
            for b in binaries:
                override = self._binary_override(overridesuite, b)
                if override is None:
                    self.warnings.append('binary:{0} is NEW.'.format(b.name))
                    new = True

        # Check all uploaded packages.
        # This is necessary to account for packages without a Package-List
        # field, really late binary-only uploads (where an unused override
        # was already removed), and for debug packages uploaded to a suite
        # without a debug suite (which are then considered as NEW).
        binaries = self.changes.binaries
        for b in binaries:
            if utils.is_in_debug_section(b.control) and suite.debug_suite is not None:
                continue
            override = self._binary_override(overridesuite, b)
            if override is None:
                self.warnings.append('binary:{0} is NEW.'.format(b.name))
                new = True

        return new

    def _check_new(self, suite, overridesuite):
        """Check if upload is NEW

        An upload is NEW if it has binary or source packages that do not have
        an override in C{overridesuite} OR if it references files ONLY in a
        tainted archive (eg. when it references files in NEW).

        Debug packages (*-dbgsym in Section: debug) are not considered as NEW
        if C{suite} has a separate debug suite.

        @rtype:  bool
        @return: C{True} if the upload is NEW, C{False} otherwise
        """
        session = self.session
        new = False

        # Check for missing overrides
        if self._check_new_binary_overrides(suite, overridesuite):
            new = True
        if self.changes.source is not None:
            override = self._source_override(overridesuite, self.changes.source)
            if override is None:
                self.warnings.append('source:{0} is NEW.'.format(self.changes.source.dsc['Source']))
                new = True

        # Check if we reference a file only in a tainted archive
        files = self.changes.files.values()
        if self.changes.source is not None:
            files.extend(self.changes.source.files.values())
        for f in files:
            query = session.query(ArchiveFile).join(PoolFile).filter(PoolFile.sha1sum == f.sha1sum)
            query_untainted = query.join(Archive).filter(Archive.tainted == False)

            in_archive = (query.first() is not None)
            in_untainted_archive = (query_untainted.first() is not None)

            if in_archive and not in_untainted_archive:
                self.warnings.append('{0} is only available in NEW.'.format(f.filename))
                new = True

        return new

    def _final_suites(self):
        session = self.session

        mapped_suites = self._mapped_suites()
        final_suites = set()

        for suite in mapped_suites:
            overridesuite = suite
            if suite.overridesuite is not None:
                overridesuite = session.query(Suite).filter_by(suite_name=suite.overridesuite).one()
            if self._check_new(suite, overridesuite):
                self.new = True
            final_suites.add(suite)

        return final_suites

    def _binary_override(self, suite, binary):
        """Get override entry for a binary

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: suite to get override for

        @type  binary: L{daklib.upload.Binary} or L{daklib.packagelist.PackageListEntry}
        @param binary: binary to get override for

        @rtype:  L{daklib.dbconn.Override} or C{None}
        @return: override for the given binary or C{None}
        """
        if suite.overridesuite is not None:
            suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()

        mapped_component = get_mapped_component(binary.component)
        if mapped_component is None:
            return None

        query = self.session.query(Override).filter_by(suite=suite, package=binary.name) \
                .join(Component).filter(Component.component_name == mapped_component.component_name) \
                .join(OverrideType).filter(OverrideType.overridetype == binary.type)

        try:
            return query.one()
        except NoResultFound:
            return None

    def _source_override(self, suite, source):
        """Get override entry for a source

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: suite to get override for

        @type  source: L{daklib.upload.Source}
        @param source: source to get override for

        @rtype:  L{daklib.dbconn.Override} or C{None}
        @return: override for the given source or C{None}
        """
        if suite.overridesuite is not None:
            suite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()

        query = self.session.query(Override).filter_by(suite=suite, package=source.dsc['Source']) \
                .join(OverrideType).filter(OverrideType.overridetype == 'dsc')

        component = source_component_from_package_list(source.package_list, suite)
        if component is not None:
            query = query.filter(Override.component == component)

        try:
            return query.one()
        except NoResultFound:
            return None

    def _binary_component(self, suite, binary, only_overrides=True):
        """get component for a binary

        By default this will only look at overrides to get the right component;
        if C{only_overrides} is C{False} this method will also look at the
        Section field.

        @type  suite: L{daklib.dbconn.Suite}

        @type  binary: L{daklib.upload.Binary}

        @type  only_overrides: bool
        @param only_overrides: only use overrides to get the right component

        @rtype: L{daklib.dbconn.Component} or C{None}
        """
        override = self._binary_override(suite, binary)
        if override is not None:
            return override.component
        if only_overrides:
            return None
        return get_mapped_component(binary.component, self.session)

    def check(self, force=False):
        """run checks against the upload

        @type  force: bool
        @param force: ignore failing forcable checks

        @rtype:  bool
        @return: C{True} if all checks passed, C{False} otherwise
        """
        # XXX: needs to be better structured.
        assert self.changes.valid_signature

        try:
            # Validate signatures and hashes before we do any real work:
            for chk in (
                    checks.SignatureAndHashesCheck,
                    checks.WeakSignatureCheck,
                    checks.SignatureTimestampCheck,
                    checks.ChangesCheck,
                    #checks.SuffixCheck,
                    checks.ExternalHashesCheck,
                    checks.SourceCheck,
                    checks.BinaryCheck,
                    checks.BinaryTimestampCheck,
                    checks.SingleDistributionCheck,
                    checks.ArchAllBinNMUCheck,
                    ):
                chk().check(self)

            final_suites = self._final_suites()
            if len(final_suites) == 0:
                self.reject_reasons.append('No target suite found. Please check your target distribution and that you uploaded to the right archive.')
                return False

            self.final_suites = final_suites

            for chk in (
                    checks.TransitionCheck,
                    checks.ACLCheck,
                    checks.NoSourceOnlyCheck,
                    checks.LintianCheck,
                    ):
                chk().check(self)

            for chk in (
                    checks.SuiteCheck,
                    checks.ACLCheck,
                    checks.SourceFormatCheck,
                    checks.SuiteArchitectureCheck,
                    checks.VersionCheck,
                    ):
                for suite in final_suites:
                    chk().per_suite_check(self, suite)

            if len(self.reject_reasons) != 0:
                return False

            self._checked = True
            return True
        except checks.Reject as e:
            self.reject_reasons.append(unicode(e))
        except Exception as e:
            self.reject_reasons.append("Processing raised an exception: {0}.\n{1}".format(e, traceback.format_exc()))
        return False

    def _install_to_suite(self, suite, source_component_func, binary_component_func, source_suites=None, extra_source_archives=None):
        """Install upload to the given suite

        @type  suite: L{daklib.dbconn.Suite}
        @param suite: suite to install the package into. This is the real suite,
                      ie. after any redirection to NEW or a policy queue

        @param source_component_func: function to get the L{daklib.dbconn.Component}
                                      for a L{daklib.upload.Source} object

        @param binary_component_func: function to get the L{daklib.dbconn.Component}
                                      for a L{daklib.upload.Binary} object

        @param source_suites: see L{daklib.archive.ArchiveTransaction.install_binary}

        @param extra_source_archives: see L{daklib.archive.ArchiveTransaction.install_binary}

        @return: tuple with two elements. The first is a L{daklib.dbconn.DBSource}
                 object for the install source or C{None} if no source was
                 included. The second is a list of L{daklib.dbconn.DBBinary}
                 objects for the installed binary packages.
        """
        # XXX: move this function to ArchiveTransaction?

        control = self.changes.changes
        changed_by = get_or_set_maintainer(control.get('Changed-By', control['Maintainer']), self.session)

        if source_suites is None:
            source_suites = self.session.query(Suite).join((VersionCheck, VersionCheck.reference_id == Suite.suite_id)).filter(VersionCheck.check == 'Enhances').filter(VersionCheck.suite == suite).subquery()

        source = self.changes.source
        if source is not None:
            component = source_component_func(source)
            db_source = self.transaction.install_source(
                self.directory,
                source,
                suite,
                component,
                changed_by,
                fingerprint=self.fingerprint
            )
        else:
            db_source = None

        db_binaries = []
        for binary in self.changes.binaries:
            copy_to_suite = suite
            if utils.is_in_debug_section(binary.control) and suite.debug_suite is not None:
                copy_to_suite = suite.debug_suite

            component = binary_component_func(binary)
            db_binary = self.transaction.install_binary(
                self.directory,
                binary,
                copy_to_suite,
                component,
                fingerprint=self.fingerprint,
                source_suites=source_suites,
                extra_source_archives=extra_source_archives
            )
            db_binaries.append(db_binary)

        if suite.copychanges:
            src = os.path.join(self.directory, self.changes.filename)
            dst = os.path.join(suite.archive.path, 'dists', suite.suite_name, self.changes.filename)
            self.transaction.fs.copy(src, dst, mode=suite.archive.mode)

        suite.update_last_changed()

        return (db_source, db_binaries)

    def _install_changes(self):
        assert self.changes.valid_signature
        control = self.changes.changes
        session = self.transaction.session
        config = Config()

        changelog_id = None
        # Only add changelog for sourceful uploads and binNMUs
        if 'source' in self.changes.architectures or re_bin_only_nmu.search(control['Version']):
            query = 'INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id'
            changelog_id = session.execute(query, {'changelog': control['Changes']}).scalar()
            assert changelog_id is not None

        db_changes = DBChange()
        db_changes.changesname = self.changes.filename
        db_changes.source = control['Source']
        db_changes.binaries = control.get('Binary', None)
        db_changes.architecture = control['Architecture']
        db_changes.version = control['Version']
        db_changes.distribution = control['Distribution']
        db_changes.urgency = control['Urgency']
        db_changes.maintainer = control['Maintainer']
        db_changes.changedby = control.get('Changed-By', control['Maintainer'])
        db_changes.date = control['Date']
        db_changes.fingerprint = self.fingerprint.fingerprint
        db_changes.changelog_id = changelog_id
        db_changes.closes = self.changes.closed_bugs

        try:
            self.transaction.session.add(db_changes)
            self.transaction.session.flush()
        except sqlalchemy.exc.IntegrityError:
            raise ArchiveException('{0} is already known.'.format(self.changes.filename))

        return db_changes

    def _install_policy(self, policy_queue, target_suite, db_changes, db_source, db_binaries):
        u = PolicyQueueUpload()
        u.policy_queue = policy_queue
        u.target_suite = target_suite
        u.changes = db_changes
        u.source = db_source
        u.binaries = db_binaries
        self.transaction.session.add(u)
        self.transaction.session.flush()

        queue_files = [self.changes.filename]
        queue_files.extend(f.filename for f in self.changes.buildinfo_files)
        for fn in queue_files:
            src = os.path.join(self.changes.directory, fn)
            dst = os.path.join(policy_queue.path, fn)
            self.transaction.fs.copy(src, dst, mode=policy_queue.change_perms)

        return u

    def try_autobyhand(self):
        """Try AUTOBYHAND

        Try to handle byhand packages automatically.

        @rtype:  list of L{daklib.upload.HashedFile}
        @return: list of remaining byhand files
        """
        assert len(self.reject_reasons) == 0
        assert self.changes.valid_signature
        assert self.final_suites is not None
        assert self._checked

        byhand = self.changes.byhand_files
        if len(byhand) == 0:
            return True

        suites = list(self.final_suites)
        assert len(suites) == 1, "BYHAND uploads must be to a single suite"
        suite = suites[0]

        cnf = Config()
        control = self.changes.changes
        automatic_byhand_packages = cnf.subtree("AutomaticByHandPackages")

        remaining = []
        for f in byhand:
            if '_' in f.filename:
                parts = f.filename.split('_', 2)
                if len(parts) != 3:
                    print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
                    remaining.append(f)
                    continue

                package, version, archext = parts
                arch, ext = archext.split('.', 1)
            else:
                parts = f.filename.split('.')
                if len(parts) < 2:
                    print "W: unexpected byhand filename {0}. No automatic processing.".format(f.filename)
                    remaining.append(f)
                    continue

                package = parts[0]
                version = '0'
                arch = 'all'
                ext = parts[-1]

            try:
                rule = automatic_byhand_packages.subtree(package)
            except KeyError:
                remaining.append(f)
                continue

            if rule['Source'] != self.changes.source_name \
                    or rule['Section'] != f.section \
                    or ('Extension' in rule and rule['Extension'] != ext):
                remaining.append(f)
                continue

            script = rule['Script']
            retcode = daklib.daksubprocess.call([script, os.path.join(self.directory, f.filename), control['Version'], arch, os.path.join(self.directory, self.changes.filename), suite.suite_name], shell=False)
            if retcode != 0:
                print "W: error processing {0}.".format(f.filename)
                remaining.append(f)

        return len(remaining) == 0

    def _install_byhand(self, policy_queue_upload, hashed_file):
        """install byhand file

        @type  policy_queue_upload: L{daklib.dbconn.PolicyQueueUpload}

        @type  hashed_file: L{daklib.upload.HashedFile}
        """
        fs = self.transaction.fs
        session = self.transaction.session
        policy_queue = policy_queue_upload.policy_queue

        byhand_file = PolicyQueueByhandFile()
        byhand_file.upload = policy_queue_upload
        byhand_file.filename = hashed_file.filename
        session.add(byhand_file)
        session.flush()

        src = os.path.join(self.directory, hashed_file.filename)
        dst = os.path.join(policy_queue.path, hashed_file.filename)
        fs.copy(src, dst, mode=policy_queue.change_perms)

        return byhand_file

    def _do_bts_versiontracking(self):
        cnf = Config()
        fs = self.transaction.fs

        btsdir = cnf.get('Dir::BTSVersionTrack')
        if btsdir is None or btsdir == '':
            return

        base = os.path.join(btsdir, self.changes.filename[:-8])

        # version history
        sourcedir = self.unpacked_source()
        if sourcedir is not None:
            fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r')
            versions = fs.create("{0}.versions".format(base), mode=0o644)
            for line in fh.readlines():
                if re_changelog_versions.match(line):
                    versions.write(line)
            fh.close()
            versions.close()

        # binary -> source mapping
        debinfo = fs.create("{0}.debinfo".format(base), mode=0o644)
        for binary in self.changes.binaries:
            control = binary.control
            source_package, source_version = binary.source
            line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version])
            print >>debinfo, line
        debinfo.close()

    def _policy_queue(self, suite):
        if suite.policy_queue is not None:
            return suite.policy_queue
        return None

    def install(self):
        """install upload

        Install upload to a suite or policy queue.  This method does B{not}
        handle uploads to NEW.

        You need to have called the C{check} method before calling this method.
        """
        assert len(self.reject_reasons) == 0
        assert self.changes.valid_signature
        assert self.final_suites is not None
        assert self._checked
        assert not self.new

        db_changes = self._install_changes()

        for suite in self.final_suites:
            overridesuite = suite
            if suite.overridesuite is not None:
                overridesuite = self.session.query(Suite).filter_by(suite_name=suite.overridesuite).one()

            policy_queue = self._policy_queue(suite)

            redirected_suite = suite
            if policy_queue is not None:
                redirected_suite = policy_queue.suite

            # source can be in the suite we install to or any suite we enhance
            source_suite_ids = set([suite.suite_id, redirected_suite.suite_id])
            for enhanced_suite_id, in self.session.query(VersionCheck.reference_id) \
                    .filter(VersionCheck.suite_id.in_(source_suite_ids)) \
                    .filter(VersionCheck.check == 'Enhances'):
                source_suite_ids.add(enhanced_suite_id)

            source_suites = self.session.query(Suite).filter(Suite.suite_id.in_(source_suite_ids)).subquery()

            source_component_func = lambda source: self._source_override(overridesuite, source).component
            binary_component_func = lambda binary: self._binary_component(overridesuite, binary, only_overrides=False)

            (db_source, db_binaries) = self._install_to_suite(redirected_suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])

            if policy_queue is not None:
                self._install_policy(policy_queue, suite, db_changes, db_source, db_binaries)

            # copy to build queues
            if policy_queue is None or policy_queue.send_to_build_queues:
                for build_queue in suite.copy_queues:
                    self._install_to_suite(build_queue.suite, source_component_func, binary_component_func, source_suites=source_suites, extra_source_archives=[suite.archive])

        self._do_bts_versiontracking()

    def install_to_new(self):
        """install upload to NEW

        Install upload to NEW.  This method does B{not} handle regular uploads
        to suites or policy queues.

        You need to have called the C{check} method before calling this method.
        """
        # Uploads to NEW are special as we don't have overrides.
        assert len(self.reject_reasons) == 0
        assert self.changes.valid_signature
        assert self.final_suites is not None

        source = self.changes.source
        binaries = self.changes.binaries
        byhand = self.changes.byhand_files

        # we need a suite to guess components
        suites = list(self.final_suites)
        assert len(suites) == 1, "NEW uploads must be to a single suite"
        suite = suites[0]

        # decide which NEW queue to use
        if suite.new_queue is None:
            new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='new').one()
        else:
            new_queue = suite.new_queue
        if len(byhand) > 0:
            # There is only one global BYHAND queue
            new_queue = self.transaction.session.query(PolicyQueue).filter_by(queue_name='byhand').one()
        new_suite = new_queue.suite


        def binary_component_func(binary):
            return self._binary_component(suite, binary, only_overrides=False)

        # guess source component
        # XXX: should be moved into an extra method
        binary_component_names = set()
        for binary in binaries:
            component = binary_component_func(binary)
            binary_component_names.add(component.component_name)
        source_component_name = None
        for c in self.session.query(Component).order_by(Component.component_id):
            guess = c.component_name
            if guess in binary_component_names:
                source_component_name = guess
                break
        if source_component_name is None:
            source_component = self.session.query(Component).order_by(Component.component_id).first()
        else:
            source_component = self.session.query(Component).filter_by(component_name=source_component_name).one()
        source_component_func = lambda source: source_component

        db_changes = self._install_changes()
        (db_source, db_binaries) = self._install_to_suite(new_suite, source_component_func, binary_component_func, source_suites=True, extra_source_archives=[suite.archive])
        policy_upload = self._install_policy(new_queue, suite, db_changes, db_source, db_binaries)

        for f in byhand:
            self._install_byhand(policy_upload, f)

        self._do_bts_versiontracking()

    def commit(self):
        """commit changes"""
        self.transaction.commit()

    def rollback(self):
        """rollback changes"""
        self.transaction.rollback()

    def __enter__(self):
        self.prepare()
        return self

    def __exit__(self, type, value, traceback):
        if self.directory is not None:
            shutil.rmtree(self.directory)
            self.directory = None
        self.changes = None
        self.transaction.rollback()
        return None