From aed272e2d6c788ff374412bcf3a34ae82f6c5414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=98=B8?= <😸@43-1.org> Date: Sat, 19 Sep 2020 13:21:48 +0200 Subject: [PATCH] process-policy: accept uploads from NEW into other policy queues It could look a bit nicer, but at least it works for now. --- dak/process_policy.py | 61 +++++++++++----- integration-tests/tests/0007-policy-queue | 88 +++++++++++++++++++++++ 2 files changed, 133 insertions(+), 16 deletions(-) create mode 100755 integration-tests/tests/0007-policy-queue diff --git a/dak/process_policy.py b/dak/process_policy.py index f3c3b7a0..d0cf10bf 100644 --- a/dak/process_policy.py +++ b/dak/process_policy.py @@ -169,8 +169,13 @@ def comment_accept(upload, srcqueue, comments, transaction): .join(Component) return query.one().component - all_target_suites = [upload.target_suite] - all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues]) + policy_queue = upload.target_suite.policy_queue + if policy_queue == srcqueue: + policy_queue = None + + all_target_suites = [upload.target_suite if policy_queue is None else policy_queue.suite] + if policy_queue is None or policy_queue.send_to_build_queues: + all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues]) throw_away_binaries = False if upload.source is not None: @@ -236,7 +241,7 @@ def comment_accept(upload, srcqueue, comments, transaction): suite.update_last_changed() # Copy .changes if needed - if upload.target_suite.copychanges: + if policy_queue is None and upload.target_suite.copychanges: src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(upload.target_suite.path, upload.changes.changesname) fs.copy(src, dst, mode=upload.target_suite.archive.mode) @@ -246,11 +251,29 @@ def comment_accept(upload, srcqueue, comments, transaction): chg = daklib.upload.Changes(upload.policy_queue.path, changesname, keyrings=[], require_signature=False) queue_files.extend(f.filename for f in chg.buildinfo_files) + # TODO: similar code exists in archive.py's `ArchiveUpload._install_policy` + if policy_queue is not None: + # register upload in policy queue + new_upload = PolicyQueueUpload() + new_upload.policy_queue = policy_queue + new_upload.target_suite = upload.target_suite + new_upload.changes = upload.changes + new_upload.source = upload.source + new_upload.binaries = upload.binaries + session.add(new_upload) + session.flush() + + # copy .changes & similar to policy queue + for fn in queue_files: + src = os.path.join(upload.policy_queue.path, fn) + dst = os.path.join(policy_queue.path, fn) + transaction.fs.copy(src, dst, mode=policy_queue.change_perms) + # Copy upload to Process-Policy::CopyDir # Used on security.d.o to sync accepted packages to ftp-master, but this # should eventually be replaced by something else. copydir = cnf.get('Process-Policy::CopyDir') or None - if copydir is not None: + if policy_queue is None and copydir is not None: mode = upload.target_suite.archive.mode if upload.source is not None: for f in [df.poolfile for df in upload.source.srcfiles]: @@ -272,10 +295,11 @@ def comment_accept(upload, srcqueue, comments, transaction): if os.path.exists(src) and not os.path.exists(dst): fs.copy(src, dst, mode=mode) - utils.process_buildinfos(upload.policy_queue.path, chg.buildinfo_files, - fs, Logger) + if policy_queue is None: + utils.process_buildinfos(upload.policy_queue.path, chg.buildinfo_files, + fs, Logger) - if upload.source is not None and not Options['No-Action']: + if policy_queue is None and upload.source is not None and not Options['No-Action']: urgency = upload.changes.urgency # As per policy 5.6.17, the urgency can be followed by a space and a # comment. Extract only the urgency from the string. @@ -285,23 +309,28 @@ def comment_accept(upload, srcqueue, comments, transaction): urgency = cnf['Urgency::Default'] UrgencyLog().log(upload.source.source, upload.source.version, urgency) - print(" ACCEPT") + if policy_queue is None: + print(" ACCEPT") + else: + print(" ACCEPT-TO-QUEUE") if not Options['No-Action']: Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) - pu = get_processed_upload(upload) - daklib.announce.announce_accept(pu) + if policy_queue is None: + pu = get_processed_upload(upload) + daklib.announce.announce_accept(pu) # TODO: code duplication. Similar code is in process-upload. # Move .changes to done now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) - for fn in queue_files: - src = os.path.join(upload.policy_queue.path, fn) - if os.path.exists(src): - dst = os.path.join(donedir, fn) - dst = utils.find_next_free(dst) - fs.copy(src, dst, mode=0o644) + if policy_queue is None: + for fn in queue_files: + src = os.path.join(upload.policy_queue.path, fn) + if os.path.exists(src): + dst = os.path.join(donedir, fn) + dst = utils.find_next_free(dst) + fs.copy(src, dst, mode=0o644) if throw_away_binaries and upload.target_suite.archive.use_morgue: morguesubdir = cnf.get("New::MorgueSubDir", 'new') diff --git a/integration-tests/tests/0007-policy-queue b/integration-tests/tests/0007-policy-queue new file mode 100755 index 00000000..61e6f72b --- /dev/null +++ b/integration-tests/tests/0007-policy-queue @@ -0,0 +1,88 @@ +#! /bin/bash +# +# © 2020, 😸 <😸@43-1.org> +# License: GPL-2+ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -e +set -u + +. ${DAK_ROOT:?}/integration-tests/common +. ${DAK_ROOT:?}/integration-tests/setup + +# setup archive +( + # add unstable with amd64 + dak admin architecture add amd64 AMD64 + dak admin suite add unstable "" codename=sid + dak admin suite-architecture add unstable source all amd64 + dak admin suite-component add unstable main contrib non-free + + # add embargoed policy queue + dak admin suite add embargoed "" codename=embargoed archive=policy + + psql -1 -d projectb <<-EOT + \set ON_ERROR_STEP + INSERT INTO policy_queue (queue_name, path, send_to_build_queues, suite_id) + VALUES ('embargoed', + '${DAKBASE}/queue/embargoed', + TRUE, + (SELECT id FROM suite WHERE suite_name = 'embargoed')); + UPDATE suite + SET policy_queue_id = (SELECT id FROM policy_queue WHERE queue_name = 'embargoed') + WHERE suite_name = 'unstable'; + EOT +) + +suite_with_contents="\ +linux-image-all 42.0-1 all +linux-image-all-signed-template 42.0-1 all +linux 42.0-1 source" + +# upload something to NEW +( + packages=$(fixture-package-dir) + import-fixture-signing-key + + dcmd cp -n -t ${DAKBASE}/tmp ${packages:?}/linux_42.0-1_amd64.changes + dak process-upload -d ${DAKBASE}/tmp --automatic + + assert-equal "dak control-suite -l embargoed" "$(dak control-suite -l embargoed)" "" + assert-equal "dak control-suite -l new" "$(dak control-suite -l new)" "${suite_with_contents}" + assert-equal "dak control-suite -l unstable" "$(dak control-suite -l unstable)" "" +) + +# accept upload form NEW into policy queue +( + echo a | dak process-new linux_42.0-1_amd64.changes + dak process-policy new + dak clean-suites + + assert-equal "dak control-suite -l embargoed" "$(dak control-suite -l embargoed)" "${suite_with_contents}" + assert-equal "dak control-suite -l new" "$(dak control-suite -l new)" "" + assert-equal "dak control-suite -l unstable" "$(dak control-suite -l unstable)" "" +) + +# accept upload from policy queue into target suite +( + mkdir ${DAKBASE}/queue/embargoed/COMMENTS + echo OK > ${DAKBASE}/queue/embargoed/COMMENTS/ACCEPT.linux_42.0-1 + dak process-policy embargoed + dak clean-suites + + assert-equal "dak control-suite -l embargoed" "$(dak control-suite -l embargoed)" "" + assert-equal "dak control-suite -l new" "$(dak control-suite -l new)" "" + assert-equal "dak control-suite -l unstable" "$(dak control-suite -l unstable)" "${suite_with_contents}" +) -- GitLab