提交 4a267a9e 编写于 作者: A Archive Administrator

[???] various (aj?) changes

上级 ea17738c
......@@ -12,58 +12,65 @@ set -e
export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars
. $SCRIPTVARS
if [ ! -e $ftpdir/Archive_Maintenance_In_Progress ]; then
cd $masterdir
for d in $DISTS; do
eval SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz`
eval PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz`
done
apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd
dists=
for d in $DISTS; do
eval NEW_SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz`
eval NEW_PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz`
old=SOURCES_$d
new=NEW_$old
if [ ${!new} -gt ${!old} ]; then
if [ -z "$dists" ]; then
dists="$d"
else
dists="$dists $d"
fi
continue
if [ -e $ftpdir/Archive_Maintenance_In_Progress ]; then
exit 0
fi
cd $masterdir
for d in $DISTS; do
eval SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz`
eval PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz`
done
apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd
dists=
for d in $DISTS; do
eval NEW_SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz`
eval NEW_PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz`
old=SOURCES_$d
new=NEW_$old
if [ ${!new} -gt ${!old} ]; then
if [ -z "$dists" ]; then
dists="$d"
else
dists="$dists $d"
fi
old=PACKAGES_$d
new=NEW_$old
if [ ${!new} -gt ${!old} ]; then
if [ -z "$dists" ]; then
dists="$d"
else
dists="$dists $d"
fi
continue
continue
fi
old=PACKAGES_$d
new=NEW_$old
if [ ${!new} -gt ${!old} ]; then
if [ -z "$dists" ]; then
dists="$d"
else
dists="$dists $d"
fi
continue
fi
done
if [ ! -z "$dists" ]; then
# setup ssh master process
ssh buildd@buildd -S $SSH_SOCKET -MN 2> /dev/null &
SSH_PID=$!
while [ ! -S $SSH_SOCKET ]; do
sleep 1
done
if [ ! -z "$dists" ]; then
# setup ssh master process
ssh buildd@buildd -S $SSH_SOCKET -MN 2> /dev/null &
SSH_PID=$!
while [ ! -S $SSH_SOCKET ]; do
sleep 1
done
trap 'kill -TERM $SSH_PID' 0
for d in $dists; do
archs=ARCHS_$d
ARCHS=${!archs}
cd /org/security.debian.org/buildd/$d
trap 'kill -TERM $SSH_PID' 0
for d in $dists; do
archs=ARCHS_$d
ARCHS=${!archs}
cd /org/security.debian.org/buildd/$d
if [ "$d" != "oldstable" ]; then
# disabled for oldstable-security by ajt 2008-01-01
for a in $ARCHS; do
quinn-diff -a /org/security.debian.org/buildd/Packages-arch-specific -A $a 2>/dev/null | ssh buildd@buildd -S $SSH_SOCKET wanna-build -d $d-security -b $a/build-db --merge-partial-quinn
ssh buildd@buildd -S $SSH_SOCKET wanna-build -d $d-security -A $a -b $a/build-db --merge-packages < Packages
done
if [ "$d" = "oldstable" ]; then
ssh buildd@bester.farm.ftbfs.de -i ~/.ssh/id_bester sleep 1
fi
done
fi
else
ssh buildd@bester.farm.ftbfs.de -i ~/.ssh/id_bester sleep 1
fi
done
fi
......@@ -10,11 +10,12 @@ export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars
# Fix overrides
rsync -ql ftp-master::indices/override\* $overridedir
# disabled by ajt 2008-01-01: requires auth
rsync --password-file /srv/non-us.debian.org/s3kr1t/rsync-password -ql security-master@ftp-master::indices/override\* $overridedir
cd $overridedir
find . -name override\*.gz -type f -maxdepth 1 -mindepth 1 | xargs gunzip -f
find . -type l -maxdepth 1 -mindepth 1 | xargs rm
find . -type l -maxdepth 1 -mindepth 1 | xargs --no-run-if-empty rm
for suite in $suites; do
case $suite in
......@@ -39,7 +40,9 @@ for suite in $suites; do
dak control-overrides -q -S -t $override_type -s $suite -c updates/$component < override.$override_suite.$component$type
fi
else
dak control-overrides -q -S -t $override_type -s $suite -c updates/$component < override.$override_suite.$component$type
# XXX removed 2007-08-16 ajt
#dak control-overrides -q -S -t $override_type -s $suite -c updates/$component < override.$override_suite.$component$type
true
fi
case $suite in
oldstable)
......
Dinstall
{
PGPKeyring "/org/keyring.debian.org/keyrings/debian-keyring.pgp";
GPGKeyring "/org/keyring.debian.org/keyrings/debian-keyring.gpg";
GPGKeyring {
"/org/keyring.debian.org/keyrings/debian-keyring.gpg";
"/org/keyring.debian.org/keyrings/debian-keyring.pgp";
};
SigningKeyring "/org/non-us.debian.org/s3kr1t/dot-gnupg/secring.gpg";
SigningPubKeyring "/org/non-us.debian.org/s3kr1t/dot-gnupg/pubring.gpg";
SigningKeyIds "6070D3A1";
......@@ -249,6 +251,7 @@ Dir
Reject "/org/security.debian.org/queue/reject/";
Unchecked "/org/security.debian.org/queue/unchecked/";
ProposedUpdates "/does/not/exist/"; // XXX fixme
OldProposedUpdates "/does/not/exist/"; // XXX fixme
Embargoed "/org/security.debian.org/queue/embargoed/";
Unembargoed "/org/security.debian.org/queue/unembargoed/";
......
......@@ -23,7 +23,7 @@
import daklib.queue, daklib.logging, daklib.utils, daklib.database
import apt_pkg, os, sys, pwd, time, re, commands
re_taint_free = re.compile(r"^['/;\-\+\.\s\w]+$");
re_taint_free = re.compile(r"^['/;\-\+\.~\s\w]+$");
Cnf = None
Options = None
......@@ -44,6 +44,7 @@ def init():
('n', "no-action", "Security-Install::Options::No-Action"),
('s', "sudo", "Security-Install::Options::Sudo"),
(' ', "no-upload", "Security-Install::Options::No-Upload"),
('u', "fg-upload", "Security-Install::Options::Foreground-Upload"),
(' ', "drop-advisory", "Security-Install::Options::Drop-Advisory"),
('A', "approve", "Security-Install::Options::Approve"),
('R', "reject", "Security-Install::Options::Reject"),
......@@ -71,6 +72,8 @@ def init():
daklib.utils.fubar("Process what?")
Upload = daklib.queue.Upload(Cnf)
if Options["No-Action"]:
Options["Sudo"] = ""
if not Options["Sudo"] and not Options["No-Action"]:
Logger = Upload.Logger = daklib.logging.Logger(Cnf, "new-security-install")
......@@ -197,9 +200,96 @@ def yes_no(prompt):
def do_upload():
if Options["No-Upload"]:
print "Not uploading as requested"
return
elif Options["Foreground-Upload"]:
actually_upload(changes)
else:
child = os.fork()
if child == 0:
actually_upload(changes)
os._exit(0)
print "Uploading in the background"
def actually_upload(changes_files):
file_list = ""
suites = {}
component_mapping = {}
for component in Cnf.SubTree("Security-Install::ComponentMappings").List():
component_mapping[component] = Cnf["Security-Install::ComponentMappings::%s" % (component)]
uploads = {}; # uploads[uri] = file_list
changesfiles = {}; # changesfiles[uri] = file_list
package_list = {} # package_list[source_name][version]
changes_files.sort(daklib.utils.changes_compare)
for changes_file in changes_files:
changes_file = daklib.utils.validate_changes_file_arg(changes_file)
# Reset variables
components = {}
upload_uris = {}
file_list = []
Upload.init_vars()
# Parse the .dak file for the .changes file
Upload.pkg.changes_file = changes_file
Upload.update_vars()
files = Upload.pkg.files
changes = Upload.pkg.changes
dsc = Upload.pkg.dsc
# We have the changes, now return if its amd64, to not upload them to ftp-master
if changes["distribution"].has_key("oldstable-security") and changes["architecture"].has_key("amd64"):
print "Not uploading amd64 oldstable-security changes to ftp-master\n"
continue
# Build the file list for this .changes file
for file in files.keys():
poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"],
daklib.utils.poolify(changes["source"], files[file]["component"]),
file)
file_list.append(poolname)
orig_component = files[file].get("original component", files[file]["component"])
components[orig_component] = ""
# Determine the upload uri for this .changes file
for component in components.keys():
upload_uri = component_mapping.get(component)
if upload_uri:
upload_uris[upload_uri] = ""
num_upload_uris = len(upload_uris.keys())
if num_upload_uris == 0:
daklib.utils.fubar("%s: No valid upload URI found from components (%s)."
% (changes_file, ", ".join(components.keys())))
elif num_upload_uris > 1:
daklib.utils.fubar("%s: more than one upload URI (%s) from components (%s)."
% (changes_file, ", ".join(upload_uris.keys()),
", ".join(components.keys())))
upload_uri = upload_uris.keys()[0]
# Update the file list for the upload uri
if not uploads.has_key(upload_uri):
uploads[upload_uri] = []
uploads[upload_uri].extend(file_list)
# Update the changes list for the upload uri
if not changesfiles.has_key(upload_uri):
changesfiles[upload_uri] = []
changesfiles[upload_uri].append(changes_file)
# Remember the suites and source name/version
for suite in changes["distribution"].keys():
suites[suite] = ""
# Remember the source name and version
if changes["architecture"].has_key("source") and \
changes["distribution"].has_key("testing"):
if not package_list.has_key(dsc["source"]):
package_list[dsc["source"]] = {}
package_list[dsc["source"]][dsc["version"]] = ""
for uri in uploads.keys():
uploads[uri].extend(changesfiles[uri])
(host, path) = uri.split(":")
file_list = " ".join(uploads[uri])
print "Uploading files to %s..." % (host)
spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list))
print "Would upload to ftp-master" # XXX
if not Options["No-Action"]:
filename = "%s/testing-processed" % (Cnf["Dir::Log"])
file = daklib.utils.open_file(filename, 'a')
for source in package_list.keys():
for version in package_list[source].keys():
file.write(" ".join([source, version])+'\n')
file.close()
def generate_advisory(template):
global changes, advisory
......@@ -317,7 +407,6 @@ def generate_advisory(template):
adv = daklib.utils.TemplateSubst(Subst, template)
return adv
def spawn(command):
if not re_taint_free.match(command):
daklib.utils.fubar("Invalid character in \"%s\"." % (command))
......@@ -342,7 +431,7 @@ def sudo(arg, fn, exit):
if advisory == None:
daklib.utils.fubar("Must set advisory name")
os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H",
"/usr/local/bin/dak new-security-install", "-"+arg, "--", advisory)
"/usr/local/bin/dak", "new-security-install", "-"+arg, "--", advisory)
else:
fn()
if exit:
......@@ -424,7 +513,7 @@ def _do_Disembargo():
for c in changes:
daklib.utils.copy(c, os.path.join(dest, c))
os.unlink(c)
k = c[:8] + ".dak"
k = c[:-8] + ".dak"
daklib.utils.copy(k, os.path.join(dest, k))
os.unlink(k)
......@@ -450,14 +539,14 @@ def _do_Reject():
aborted = Upload.do_reject()
if not aborted:
os.unlink(c[:-8]+".katie")
os.unlink(c[:-8]+".dak")
for f in files:
Upload.projectB.query(
"DELETE FROM queue_build WHERE filename = '%s'" % (f))
os.unlink(f)
print "Updating buildd information..."
spawn("/org/security.debian.org/katie/cron.buildd-security")
spawn("/org/security.debian.org/dak/config/debian-security/cron.buildd")
adv_file = "./advisory.%s" % (advisory)
if os.path.exists(adv_file):
......
......@@ -83,7 +83,11 @@ def recheck():
and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
found = 0
for q in ["Accepted", "Embargoed", "Unembargoed"]:
if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
found = 1
if not found:
reject("no source found for %s %s (%s)." % (source_package, source_version, file))
# Version and file overwrite checks
......@@ -92,9 +96,9 @@ def recheck():
elif files[file]["type"] == "dsc":
reject(Upload.check_source_against_db(file))
(reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
reject(reject_msg)
reject(reject_msg, "")
if reject_message:
if reject_message.find("Rejected") != -1:
answer = "XXX"
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
......@@ -122,71 +126,6 @@ def recheck():
################################################################################
def determine_new (changes, files):
new = {}
# Build up a list of potentially new things
for file in files.keys():
f = files[file]
# Skip byhand elements
if f["type"] == "byhand":
continue
pkg = f["package"]
priority = f["priority"]
section = f["section"]
# FIXME: unhardcode
if section == "non-US/main":
section = "non-US"
type = get_type(f)
component = f["component"]
if type == "dsc":
priority = "source"
if not new.has_key(pkg):
new[pkg] = {}
new[pkg]["priority"] = priority
new[pkg]["section"] = section
new[pkg]["type"] = type
new[pkg]["component"] = component
new[pkg]["files"] = []
else:
old_type = new[pkg]["type"]
if old_type != type:
# source gets trumped by deb or udeb
if old_type == "dsc":
new[pkg]["priority"] = priority
new[pkg]["section"] = section
new[pkg]["type"] = type
new[pkg]["component"] = component
new[pkg]["files"].append(file)
if f.has_key("othercomponents"):
new[pkg]["othercomponents"] = f["othercomponents"]
for suite in changes["suite"].keys():
suite_id = daklib.database.get_suite_id(suite)
for pkg in new.keys():
component_id = daklib.database.get_component_id(new[pkg]["component"])
type_id = daklib.database.get_override_type_id(new[pkg]["type"])
q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
ql = q.getresult()
if ql:
for file in new[pkg]["files"]:
if files[file].has_key("new"):
del files[file]["new"]
del new[pkg]
if changes["suite"].has_key("stable"):
print "WARNING: overrides will be added for stable!"
if changes["suite"].has_key("oldstable"):
print "WARNING: overrides will be added for OLDstable!"
for pkg in new.keys():
if new[pkg].has_key("othercomponents"):
print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
return new
################################################################################
def indiv_sg_compare (a, b):
"""Sort by source name, source, version, 'have source', and
finally by filename."""
......@@ -320,25 +259,8 @@ class Priority_Completer:
################################################################################
def check_valid (new):
for pkg in new.keys():
section = new[pkg]["section"]
priority = new[pkg]["priority"]
type = new[pkg]["type"]
new[pkg]["section id"] = daklib.database.get_section_id(section)
new[pkg]["priority id"] = daklib.database.get_priority_id(new[pkg]["priority"])
# Sanity checks
if (section == "debian-installer" and type != "udeb") or \
(section != "debian-installer" and type == "udeb"):
new[pkg]["section id"] = -1
if (priority == "source" and type != "dsc") or \
(priority != "source" and type == "dsc"):
new[pkg]["priority id"] = -1
################################################################################
def print_new (new, indexed, file=sys.stdout):
check_valid(new)
daklib.queue.check_valid(new)
broken = 0
index = 0
for pkg in new.keys():
......@@ -366,24 +288,6 @@ def print_new (new, indexed, file=sys.stdout):
################################################################################
def get_type (f):
# Determine the type
if f.has_key("dbtype"):
type = f["dbtype"]
elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
type = "dsc"
else:
daklib.utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type))
# Validate the override type
type_id = daklib.database.get_override_type_id(type)
if type_id == -1:
daklib.utils.fubar("invalid type (%s) for new. Say wha?" % (type))
return type
################################################################################
def index_range (index):
if index == 1:
return "1"
......@@ -721,7 +625,7 @@ def do_new():
done = 0
while not done:
# Find out what's new
new = determine_new(changes, files)
new = daklib.queue.determine_new(changes, files, projectB)
if not new:
break
......@@ -796,10 +700,11 @@ def init():
Arguments = [('a',"automatic","Process-New::Options::Automatic"),
('h',"help","Process-New::Options::Help"),
('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
('n',"no-action","Process-New::Options::No-Action")]
for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
if not Cnf.has_key("Process-New::Options::%s" % (i)):
Cnf["Process-New::Options::%s" % (i)] = ""
......@@ -871,26 +776,42 @@ def do_byhand():
################################################################################
def get_accept_lock():
retry = 0
while retry < 10:
try:
lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
retry = 10
except OSError, e:
if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
retry += 1
if (retry >= 10):
daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
else:
print("Unable to get accepted lock (try %d of 10)" % retry)
time.sleep(60)
else:
raise
def move_to_dir (dest, perms=0660, changesperms=0664):
daklib.utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
file_keys = Upload.pkg.files.keys()
for file in file_keys:
daklib.utils.move (file, dest, perms=perms)
def do_accept():
print "ACCEPT"
if not Options["No-Action"]:
retry = 0
while retry < 10:
try:
lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
retry = 10
except OSError, e:
if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
retry += 1
if (retry >= 10):
daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
else:
print("Unable to get accepted lock (try %d of 10)" % retry)
time.sleep(60)
else:
raise
get_accept_lock()
(summary, short_summary) = Upload.build_summaries()
Upload.accept(summary, short_summary)
if Cnf.FindB("Dinstall::SecurityQueueHandling"):
Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
move_to_dir(Cnf["Dir::Queue::Embargoed"])
Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
# Check for override disparities
Upload.Subst["__SUMMARY__"] = summary
else:
Upload.accept(summary, short_summary)
os.unlink(Upload.pkg.changes_file[:-8]+".dak")
os.unlink(Cnf["Process-New::AcceptedLockFile"])
......@@ -942,6 +863,59 @@ def end():
################################################################################
def do_comments(dir, opref, npref, line, fn):
for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
lines = open("%s/%s" % (dir, comm)).readlines()
if len(lines) == 0 or lines[0] != line + "\n": continue
changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
and x.endswith(".changes") ]
changes_files = sort_changes(changes_files)
for f in changes_files:
f = daklib.utils.validate_changes_file_arg(f, 0)
if not f: continue
print "\n" + f
fn(f, "".join(lines[1:]))
if opref != npref and not Options["No-Action"]:
newcomm = npref + comm[len(opref):]
os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
################################################################################
def comment_accept(changes_file, comments):
Upload.pkg.changes_file = changes_file
Upload.init_vars()
Upload.update_vars()
Upload.update_subst()
files = Upload.pkg.files
if not recheck():
return # dak wants to REJECT, crap
(new, byhand) = check_status(files)
if not new and not byhand:
do_accept()
################################################################################
def comment_reject(changes_file, comments):
Upload.pkg.changes_file = changes_file
Upload.init_vars()
Upload.update_vars()
Upload.update_subst()
files = Upload.pkg.files
if not recheck():
pass # dak has its own reasons to reject as well, which is fine
reject(comments)
print "REJECT\n" + reject_message,
if not Options["No-Action"]:
Upload.do_reject(0, reject_message)
os.unlink(Upload.pkg.changes_file[:-8]+".dak")
################################################################################
def main():
changes_files = init()
if len(changes_files) > 50:
......@@ -956,12 +930,20 @@ def main():
else:
Upload.Subst["__BCC__"] = bcc
for changes_file in changes_files:
changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
if not changes_file:
continue
print "\n" + changes_file
do_pkg (changes_file)
commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
if commentsdir:
if changes_files != []:
sys.stderr.write("Can't specify any changes files if working with comments-dir")
sys.exit(1)
do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
else:
for changes_file in changes_files:
changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
if not changes_file:
continue
print "\n" + changes_file
do_pkg (changes_file)
end()
......
......@@ -84,9 +84,6 @@ def do_upload(changes_files):
if changes["architecture"].has_key("amd64"):
print "Not uploading amd64 part to ftp-master\n"
continue
if changes["distribution"].has_key("oldstable"):
print "Not uploading oldstable-security changes to ftp-master\n"
continue
# Build the file list for this .changes file
for file in files.keys():
poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"],
......@@ -328,6 +325,9 @@ def spawn(command):
def main():
print "Disabled. See your team@security email, and/or contact aj on OFTC."
sys.exit(1)
(advisory_number, changes_files) = init()
if not Options["No-Action"]:
......@@ -340,7 +340,7 @@ def main():
os.chdir(Cnf["Dir::Queue::Accepted"])
print "Installing packages into the archive..."
spawn("dak process-accepted -pa %s" % (Cnf["Dir::Dak"], " ".join(changes_files)))
spawn("dak process-accepted -pa %s" % (" ".join(changes_files)))
os.chdir(Cnf["Dir::Dak"])
print "Updating file lists for apt-ftparchive..."
spawn("dak make-suite-file-list")
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册