From bb8283ad6274e461fc78a00b2a52c6594c53e13d Mon Sep 17 00:00:00 2001 From: James Troup Date: Tue, 5 Dec 2000 04:27:48 +0000 Subject: [PATCH] life sucks --- TODO | 14 ++-- apt.conf-non-US | 2 +- claire.py | 107 ++++++++++++++++++++++++++ contrib/fix.9 | 105 ++++++++++++++++++++++++++ db_access.py | 4 +- heidi | 4 +- jenna | 31 +++++--- katie | 189 +++++++++++++++++++++++++++++++++++++--------- katie.conf-non-US | 1 + tea | 69 +++++++++++++++++ utils.py | 12 +-- 11 files changed, 474 insertions(+), 64 deletions(-) create mode 100755 claire.py create mode 100755 contrib/fix.9 create mode 100755 tea diff --git a/TODO b/TODO index a486b0f3..acfc5c26 100644 --- a/TODO +++ b/TODO @@ -2,18 +2,14 @@ Show Stopper ------------ o finish new cron.daily file for auric - o finish rhona + o claire needs to know about sections on non-non-US o Testing... lots of it. - o jenna needs to munge files in stable - Non-Show Stopper ---------------- - o fix heimdal at some point on non-US. - o CD building scripts need fixing o charisma needs to do version compares @@ -21,9 +17,6 @@ Non-Show Stopper o Optimize all the queries by using EXAMINE and building some INDEXs. o enclose all the setting SQL stuff in transactions (mostly done). - o ?assumes running in incoming? [needs to be runnable in p-u too] - o changelog udpate stuff for point releases. - == o need a poolifier that will poolify X mb a day.. (catherine) @@ -36,6 +29,11 @@ Non-Show Stopper o check errors on apt_pkg calls so we don't bomb out on daily runs (?) o check to see if mutli-component binary packages from a single component source are supported o need a way to sync katie.conf and the DB for things like architecture + o dinstall should remove != stable on stable install, not just f + u + o Should probably use $ in ~ compares for files. + o jenna needs to validate arguments! + o jenna's untochable flag doesn't stop adding new packages + o db_access' get_files needs to use exceptions not this None, > 0, < 0 return val BS =================================================================================================== diff --git a/apt.conf-non-US b/apt.conf-non-US index ca4bfd9e..ef7f1190 100644 --- a/apt.conf-non-US +++ b/apt.conf-non-US @@ -17,7 +17,7 @@ Default TreeDefault { - Contents::Header "/org/non-us.debian.org/scripts/masterfiles/Contents.top"; + Contents::Header "/org/non-us.debian.org/katie/Contents.top"; }; tree "dists/stable/non-US" diff --git a/claire.py b/claire.py new file mode 100755 index 00000000..1c3784c6 --- /dev/null +++ b/claire.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +# 'Fix' stable to make debian-cd and dpkg -BORGiE users happy +# Copyright (C) 2000 James Troup +# $Id: claire.py,v 1.1 2000-12-05 04:27:48 troup Exp $ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# "Look around... leaves are brown... and the sky's a hazy shade of winter, +# Look around... leaves are brown... there's a patch of snow on the ground." +# -- Simon & Garfunkel / 'A Hazy Shade' + +################################################################################ + +import pg, sys, os, string +import utils, db_access +import apt_pkg; + +################################################################################ + +Cnf = None; +projectB = None; + +# Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'. +# Returns fixed 'src' +def clean_symlink (src, dest, root): + src = string.replace(src, root, '', 1); + dest = string.replace(dest, root, '', 1); + dest = os.path.dirname(dest); + new_src = ''; + for i in xrange(len(string.split(dest, '/'))): + new_src = new_src + '../'; + return new_src + src + +################################################################################ + +def find_dislocated_stable(Cnf, projectB): + dislocated_files = {} + + # Source + q = projectB.query("SELECT su.suite_name, c.name, s.id FROM suite su, src_associations sa, source s, files f, component c, location l WHERE su.suite_name = 'stable' AND sa.suite = su.id AND sa.source = s.id AND f.id = s.file AND f.location = l.id AND (l.component = c.id OR (l.component = NULL AND c.name = 'non-US/main')) AND NOT (f.filename ~ '^potato/');") + for i in q.getresult(): + q = projectB.query("SELECT l.path, f.filename, f.id FROM source s, files f, location l, dsc_files df WHERE s.id = %d AND df.source = %d AND f.id = df.file AND f.location = l.id AND NOT (f.filename ~ '^potato/')" % (i[2], i[2])); + for j in q.getresult(): + src = j[0]+j[1] + dest = Cnf["Dir::RootDir"]+'dists/'+i[0]+'/'+i[1]+'/source/'+os.path.basename(j[1]); + src = clean_symlink(src, dest, Cnf["Dir::RootDir"]); + if not os.path.exists(dest): + if Cnf.Find("Claire::Options::Verbose"): + print src+' -> '+dest + os.symlink(src, dest); + dislocated_files[j[2]] = dest; + + # Binary + q = projectB.query("SELECT su.suite_name, c.name, a.arch_string, b.package, b.version, l.path, f.filename, f.id FROM suite su, bin_associations ba, binaries b, files f, component c, architecture a, location l WHERE ba.suite = su.id AND su.suite_name = 'stable' AND ba.bin = b.id AND f.id = b.file AND f.location = l.id AND (l.component = c.id OR (l.component = NULL and c.name = 'non-US/main')) AND b.architecture = a.id AND NOT (f.filename ~ '^potato/');"); + for i in q.getresult(): + src = i[5]+i[6] + dest = Cnf["Dir::RootDir"]+'dists/'+i[0]+'/'+i[1]+'/binary-'+i[2]+'/'+i[3]+'_'+utils.re_no_epoch.sub('', i[4])+'.deb' + src = clean_symlink(src, dest, Cnf["Dir::RootDir"]); + if not os.path.exists(dest): + if Cnf.Find("Claire::Options::Verbose"): + print src+' -> '+dest + os.symlink(src, dest); + dislocated_files[i[7]] = dest; + + return dislocated_files + +################################################################################ + +def main (): + global Cnf, projectB; + + apt_pkg.init(); + + Cnf = apt_pkg.newConfiguration(); + apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); + + Arguments = [('d',"debug","Claire::Options::Debug", "IntVal"), + ('h',"help","Claire::Options::Help"), + ('v',"verbose","Claire::Options::Verbose"), + ('V',"version","Claire::Options::Version")]; + + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + + projectB = pg.connect('projectb', 'localhost'); + + db_access.init(Cnf, projectB); + + find_dislocated_stable(Cnf, projectB); + +####################################################################################### + +if __name__ == '__main__': + main() + diff --git a/contrib/fix.9 b/contrib/fix.9 new file mode 100755 index 00000000..b461b9b0 --- /dev/null +++ b/contrib/fix.9 @@ -0,0 +1,105 @@ +#!/usr/bin/env python + +# Fix for bug in katie where dsc_files was initialized from changes and not dsc +# Copyright (C) 2000 James Troup +# $Id: fix.9,v 1.1 2000-12-05 04:27:48 troup Exp $ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# "Look around... leaves are brown... and the sky is hazy shade of winter, +# Look around... leaves are brown... there's a patch of snow on the ground." +# -- Simon & Garfunkel / 'A Hazy Shade' + +################################################################################ + +import pg, sys, os, string, stat +import utils, db_access +import apt_pkg; + +################################################################################ + +Cnf = None; +projectB = None; + +################################################################################ + +def main (): + global Cnf, projectB; + + apt_pkg.init(); + + Cnf = apt_pkg.newConfiguration(); + apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); + + Arguments = [('d',"debug","Claire::Options::Debug", "IntVal"), + ('h',"help","Claire::Options::Help"), + ('v',"version","Claire::Options::Version")]; + + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + + projectB = pg.connect('projectb', 'localhost'); + + db_access.init(Cnf, projectB); + + for dsc_file in sys.stdin.readlines(): + dsc_file = dsc_file[:-1]; + base_dsc_file = os.path.basename(dsc_file); + print dsc_file + dsc = utils.parse_changes(dsc_file); + dsc_files = utils.build_file_list(dsc, 1); + q = projectB.query("SELECT s.id, l.id, l.path FROM source s, location l, files f WHERE s.source = '%s' AND s.version = '%s' AND f.id = s.file AND f.location = l.id" % (dsc["source"], dsc["version"])); + ql = q.getresult(); + if ql == [] or len(ql) > 1: + print " EEEEEEEEEEEEEEK!!!" + print " ",base_dsc_file + source_id = ql[0][0]; + location_id = ql[0][1]; + location = ql[0][2]; + dsc_files[base_dsc_file] = {}; + dsc_files[base_dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE]; + dsc_files[base_dsc_file]["md5sum"] = apt_pkg.md5sum(utils.open_file(dsc_file,"r")); + q = projectB.query("SELECT f.filename FROM dsc_files df, source s, files f WHERE s.id = '%s' AND df.source = s.id AND f.id = df.file" % (source_id)); + for i in q.getresult(): + file = os.path.basename(i[0]); + if not dsc_files.has_key(file): + if file != base_dsc_file: + print " MWAAAP! MWAAP! Can't find %s!" % (file) + else: + del dsc_files[file]; + for i in dsc_files.keys(): + filename = os.path.dirname(dsc_file) + '/' + i; + if not os.path.exists(filename): + print " MWAAP!!!!!!!!!!!!!" + print filename + filename = string.replace(filename, location, ''); + #print " filename: ",filename + #print " size: ", dsc_files[i]["size"] + #print " md5sum: ",dsc_files[i]["md5sum"] + #print " location_id: ", location_id + files_id = db_access.get_files_id(filename, repr(dsc_files[i]["size"]), dsc_files[i]["md5sum"], location_id); + if files_id < 0 or files_id == None: + print " BORK!!!!!!!!!!!!" + print " ",filename + else: + foo = 1 + #print "INSERT INTO dsc_files (source, file) VALUES ('%s', '%s')" % (source_id, files_id); + projectB.query("INSERT INTO dsc_files (source, file) VALUES ('%s', '%s')" % (source_id, files_id)); + print " doh:",i + +####################################################################################### + +if __name__ == '__main__': + main() + diff --git a/db_access.py b/db_access.py index 0538971d..8760d91b 100644 --- a/db_access.py +++ b/db_access.py @@ -1,6 +1,6 @@ # DB access fucntions # Copyright (C) 2000 James Troup -# $Id: db_access.py,v 1.2 2000-11-27 03:15:26 troup Exp $ +# $Id: db_access.py,v 1.3 2000-12-05 04:27:48 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -154,7 +154,7 @@ def get_files_id (filename, size, md5sum, location_id): cache_key = "%s~%d" % (filename, location_id); if files_id_cache.has_key(cache_key): - return files_id_cache[files] + return files_id_cache[cache_key] q = projectB.query("SELECT id, size, md5sum FROM files WHERE filename = '%s' AND location = %d" % (filename, location_id)); ql = q.getresult(); diff --git a/heidi b/heidi index b34f951e..2ce00e7b 100755 --- a/heidi +++ b/heidi @@ -2,7 +2,7 @@ # Manipulate suite tags # Copyright (C) 2000 James Troup -# $Id: heidi,v 1.1 2000-11-24 00:20:11 troup Exp $ +# $Id: heidi,v 1.2 2000-12-05 04:27:48 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -71,7 +71,7 @@ def process_file (file, suite_id, action): if architecture == "source": q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version)) else: - q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)) + q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)) ql = q.getresult(); if ql == []: diff --git a/jenna b/jenna index 11b5c91e..b186ff78 100755 --- a/jenna +++ b/jenna @@ -2,7 +2,7 @@ # Generate file list which is then fed to apt-ftparchive to generate Packages and Sources files # Copyright (C) 2000 James Troup -# $Id: jenna,v 1.1 2000-11-24 00:20:11 troup Exp $ +# $Id: jenna,v 1.2 2000-12-05 04:27:48 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,21 +28,21 @@ import pg, string, os, sys import apt_pkg -import db_access, utils +import db_access, utils, claire projectB = None Cnf = None -def generate_src_list(suite, component, output): +def generate_src_list(suite, component, output, dislocated_files): sources = {} suite_id = db_access.get_suite_id(suite); if component == "-": - q = projectB.query("SELECT s.source, s.version, l.path, f.filename, s.id FROM source s, src_associations sa, location l, files f WHERE sa.source = s.id AND sa.suite = '%d' AND l.id = f.location AND s.file = f.id" + q = projectB.query("SELECT s.source, s.version, l.path, f.filename, s.id, f.id FROM source s, src_associations sa, location l, files f WHERE sa.source = s.id AND sa.suite = '%d' AND l.id = f.location AND s.file = f.id" % (suite_id)); else: - q = projectB.query("SELECT s.source, s.version, l.path, f.filename, s.id FROM source s, src_associations sa, location l, component c, files f WHERE lower(c.name) = '%s' AND (c.id = l.component OR l.component = NULL) AND sa.source = s.id AND sa.suite = '%d' AND l.id = f.location AND s.file = f.id" + q = projectB.query("SELECT s.source, s.version, l.path, f.filename, s.id, f.id FROM source s, src_associations sa, location l, component c, files f WHERE lower(c.name) = '%s' AND (c.id = l.component OR l.component = NULL) AND sa.source = s.id AND sa.suite = '%d' AND l.id = f.location AND s.file = f.id" % (component, suite_id)); entries = q.getresult(); for entry in entries: @@ -51,6 +51,9 @@ def generate_src_list(suite, component, output): filename = entry[2]+entry[3]; id = entry[4] add_new = 0 + file_id = entry[5]; + if dislocated_files.has_key(file_id): + filename = dislocated_files[file_id]; if os.path.exists(filename): if sources.has_key(source): if apt_pkg.VersionCompare(sources[source]["version"], version) == -1: @@ -81,15 +84,15 @@ def generate_src_list(suite, component, output): for source in source_keys: output.write(sources[source]["filename"]+'\n') -def generate_bin_list(suite, component, architecture, output, type): +def generate_bin_list(suite, component, architecture, output, type, dislocated_files): packages = {} suite_id = db_access.get_suite_id(suite); if component == "-": - q = projectB.query("SELECT b.package, b.version, l.path, f.filename, b.id FROM architecture a, binaries b, bin_associations ba, location l, files f WHERE ( a.arch_string = '%s' OR a.arch_string = 'all' ) AND a.id = b.architecture AND ba.bin = b.id AND ba.suite = '%d' AND l.id = f.location AND b.file = f.id AND b.type = '%s'" % (architecture, suite_id, type)); + q = projectB.query("SELECT b.package, b.version, l.path, f.filename, b.id, f.id FROM architecture a, binaries b, bin_associations ba, location l, files f WHERE ( a.arch_string = '%s' OR a.arch_string = 'all' ) AND a.id = b.architecture AND ba.bin = b.id AND ba.suite = '%d' AND l.id = f.location AND b.file = f.id AND b.type = '%s'" % (architecture, suite_id, type)); else: - q = projectB.query("SELECT b.package, b.version, l.path, f.filename, b.id FROM architecture a, binaries b, bin_associations ba, location l, component c, files f WHERE lower(c.name) = '%s' AND (c.id = l.component OR l.component = NULL) AND (a.arch_string = '%s' OR a.arch_string = 'all') AND a.id = b.architecture AND ba.bin = b.id AND ba.suite = '%d' AND l.id = f.location AND b.file = f.id AND b.type = '%s'" % (component, architecture, suite_id, type)); + q = projectB.query("SELECT b.package, b.version, l.path, f.filename, b.id, f.id FROM architecture a, binaries b, bin_associations ba, location l, component c, files f WHERE lower(c.name) = '%s' AND (c.id = l.component OR l.component = NULL) AND (a.arch_string = '%s' OR a.arch_string = 'all') AND a.id = b.architecture AND ba.bin = b.id AND ba.suite = '%d' AND l.id = f.location AND b.file = f.id AND b.type = '%s'" % (component, architecture, suite_id, type)); entries = q.getresult(); for entry in entries: package = entry[0] @@ -97,6 +100,9 @@ def generate_bin_list(suite, component, architecture, output, type): filename = entry[2]+entry[3]; id = entry[4] add_new = 0 + file_id = entry[5]; + if dislocated_files.has_key(file_id): + filename = dislocated_files[file_id]; # Hack to handle screwed up sid distro [FIXME: this may have issues, remove ASAP] if not os.path.exists(filename): @@ -138,6 +144,7 @@ def generate_bin_list(suite, component, architecture, output, type): def main(): global Cnf, projectB; + dislocated_files = {}; projectB = pg.connect('projectb', 'localhost'); @@ -162,6 +169,8 @@ def main(): Cnf["Jenna::Options::Suite"] = string.join(Cnf.SubTree("Suite").List()); for suite in string.split(Cnf["Jenna::Options::Suite"]): suite = string.lower(suite); + if suite == 'stable': + dislocated_files = claire.find_dislocated_stable(Cnf, projectB); components = Cnf["Jenna::Options::Component"]; if not Cnf.has_key("Suite::%s::Components" % (suite)): components = "-"; @@ -179,17 +188,17 @@ def main(): if architecture == "source": print "Processing dists/%s/%s/%s..." % (suite, component, architecture); output = utils.open_file("%s/%s_%s_%s.list" % (Cnf["Dir::ListsDir"], suite, component, architecture), "w") - generate_src_list(suite, component, output); + generate_src_list(suite, component, output, dislocated_files); output.close(); else: print "Processing dists/%s/%s/binary-%s..." % (suite, component, architecture); output = utils.open_file("%s/%s_%s_binary-%s.list" % (Cnf["Dir::ListsDir"], suite, component, architecture), "w"); - generate_bin_list(suite, component, architecture, output, "deb"); + generate_bin_list(suite, component, architecture, output, "deb", dislocated_files); output.close(); if component == "main": # FIXME: must be a cleaner way to say debian-installer is main only? print "Processing dists/%s/%s/debian-installer/binary-%s..." % (suite,component, architecture); output = utils.open_file("%s/%s_%s_debian-installer_binary-%s.list" % (Cnf["Dir::ListsDir"], suite, component, architecture), "w"); - generate_bin_list(suite, component, architecture, output, "udeb"); + generate_bin_list(suite, component, architecture, output, "udeb", dislocated_files); output.close(); if __name__ == '__main__': diff --git a/katie b/katie index 7624d6c4..3656f428 100755 --- a/katie +++ b/katie @@ -2,7 +2,7 @@ # Installs Debian packaes # Copyright (C) 2000 James Troup -# $Id: katie,v 1.8 2000-12-01 22:09:14 troup Exp $ +# $Id: katie,v 1.9 2000-12-05 04:27:48 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -118,7 +118,7 @@ def check_signature (filename): def read_override_file (filename, suite, component): global overrides; - + file = utils.open_file(filename, 'r'); for line in file.readlines(): line = string.strip(utils.re_comments.sub('', line)) @@ -229,12 +229,19 @@ def check_changes(filename): # Handle uploads to stable if changes["distribution"].has_key("stable"): - # If running from within proposed-updates kill non-stable distributions + # If running from within proposed-updates; assume an install to stable if string.find(os.getcwd(), 'proposed-updates') != -1: + # FIXME: should probably remove anything that != stable for i in ("frozen", "unstable"): if changes["distribution"].has_key(i): reject_message = reject_message + "Removing %s from distribution list.\n" del changes["distribution"][i] + changes["stable upload"] = 1; + # If we can't find a file from the .changes; assume it's a package already in the pool and move into the pool + file = files.keys()[0]; + if os.access(file, os.R_OK) == 0: + pool_dir = Cnf["Dir::PoolDir"] + '/' + utils.poolify(changes["source"], files[file]["component"]); + os.chdir(pool_dir); # Otherwise (normal case) map stable to updates else: reject_message = reject_message + "Mapping stable to updates.\n"; @@ -376,9 +383,10 @@ def check_files(): reject_message = reject_message + "Rejected" reject_message = reject_message + ": %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"]) # Check for existing copies of the file - q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"])) - if q.getresult() != []: - reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file) + if not changes.has_key("stable upload"): + q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"])) + if q.getresult() != []: + reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file) # Find any old .dsc files elif files[file]["type"] == "dsc": @@ -444,23 +452,13 @@ def check_dsc (): actual_md5 = files[dsc_file]["md5sum"] found = "%s in incoming" % (dsc_file) # Check the file does not already exist in the archive - q = projectB.query("SELECT f.id FROM files f, location l WHERE f.filename ~ '/%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); - if q.getresult() != []: - reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file) + if not changes.has_key("stable upload"): + q = projectB.query("SELECT f.id FROM files f, location l WHERE f.filename ~ '%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); + if q.getresult() != []: + reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file) elif dsc_file[-12:] == ".orig.tar.gz": - # Check in Incoming - # See comment above process_it() for explanation... - if os.access(dsc_file, os.R_OK) != 0: - files[dsc_file] = {}; - files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE]; - files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"]; - files[dsc_file]["section"] = files[file]["section"]; - files[dsc_file]["priority"] = files[file]["priority"]; - files[dsc_file]["component"] = files[file]["component"]; - reprocess = 1; - return 1; # Check in the pool - q = projectB.query("SELECT l.path, f.filename, l.type, f.id FROM files f, location l WHERE f.filename ~ '/%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); + q = projectB.query("SELECT l.path, f.filename, l.type, f.id FROM files f, location l WHERE f.filename ~ '%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); ql = q.getresult(); if len(ql) > 0: old_file = ql[0][0] + ql[0][1]; @@ -471,8 +469,21 @@ def check_dsc (): if suite_type == "legacy" or suite_type == "legacy-mixed": orig_tar_id = ql[0][3]; else: - reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file); - continue; + # Not there? Check in Incoming... + # [See comment above process_it() for explanation + # of why this is necessary...] + if os.access(dsc_file, os.R_OK) != 0: + files[dsc_file] = {}; + files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE]; + files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"]; + files[dsc_file]["section"] = files[file]["section"]; + files[dsc_file]["priority"] = files[file]["priority"]; + files[dsc_file]["component"] = files[file]["component"]; + reprocess = 1; + return 1; + else: + reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file); + continue; else: reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming." % (file, dsc_file); continue; @@ -594,6 +605,11 @@ def action (changes_filename): def install (changes_filename, summary, short_summary): global install_count, install_bytes + + # Stable uploads are a special case + if changes.has_key("stable upload"): + stable_install (changes_filename, summary, short_summary); + return; print "Installing." @@ -614,7 +630,6 @@ def install (changes_filename, summary, short_summary): dsc_location_id = files[file]["location id"]; if not files[file]["files id"]: files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id) - dsc_file_id = files[file]["files id"] projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)" % (package, version, maintainer_id, files[file]["files id"])) @@ -622,15 +637,17 @@ def install (changes_filename, summary, short_summary): suite_id = db_access.get_suite_id(suite); projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id)) - - # Add the .diff.gz and {.orig,}.tar.gz files to the DB (files and dsc_files) - for file in files.keys(): - if files[file]["type"] == "diff.gz" or files[file]["type"] == "orig.tar.gz" or files[file]["type"] == "tar.gz": - if not files[file]["files id"]: - filename = files[file]["pool name"] + file; - files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"]) - projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"])); - + # Add the source files to the DB (files and dsc_files) + projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files[file]["files id"])); + for dsc_file in dsc_files.keys(): + filename = files[file]["pool name"] + dsc_file; + files_id = db_access.get_files_id(filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], files[file]["location id"]); + # FIXME: needs to check for -1/-2 and or handle exception + if files_id == None: + files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], files[file]["location id"]); + projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id)); + + # Add the .deb files to the DB for file in files.keys(): if files[file]["type"] == "deb": @@ -723,7 +740,98 @@ Subject: %s INSTALLED Installing: %s -%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes_filename, reject_message, summary, installed_footer) +%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, summary, installed_footer) + utils.send_mail (mail_message, "") + announce (short_summary, 1) + +##################################################################################################################### + +def stable_install (changes_filename, summary, short_summary): + global install_count, install_bytes + + print "Installing to stable." + + archive = utils.where_am_i(); + + # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed. + projectB.query("BEGIN WORK"); + + # Add the .dsc file to the DB + for file in files.keys(): + if files[file]["type"] == "dsc": + package = dsc["source"] + version = dsc["version"] # NB: not files[file]["version"], that has no epoch + q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version)) + ql = q.getresult() + if ql == []: + sys.stderr.write("INTERNAL ERROR: couldn't find '%s' (%s) in source table.\n" % (package, version)); + sys.exit(1); + source_id = ql[0][0]; + suite_id = db_access.get_suite_id('proposed-updates'); + projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id)); + suite_id = db_access.get_suite_id('stable'); + projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id)); + + # Add the .deb files to the DB + for file in files.keys(): + if files[file]["type"] == "deb": + package = files[file]["package"] + version = files[file]["version"] + architecture = files[file]["architecture"] + q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)) + ql = q.getresult() + if ql == []: + sys.stderr.write("INTERNAL ERROR: couldn't find '%s' (%s for %s architecture) in binaries table.\n" % (package, version, architecture)); + sys.exit(1); + binary_id = ql[0][0]; + suite_id = db_access.get_suite_id('proposed-updates'); + projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)); + suite_id = db_access.get_suite_id('stable'); + projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id)); + + projectB.query("COMMIT WORK"); + + utils.move (changes_filename, Cnf["Rhona::Morgue"] + os.path.basename(changes_filename)); + + # Update the Stable ChangeLog file + + new_changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog"; + changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog"; + if os.path.exists(new_changelog_filename): + os.unlink (new_changelog_filename); + + new_changelog = utils.open_file(new_changelog_filename, 'w'); + for file in files.keys(): + if files[file]["type"] == "deb": + new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file)); + elif re_issource.match(file) != None: + new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file)); + else: + new_changelog.write("%s\n" % (file)); + chop_changes = re_fdnic.sub("\n", changes["changes"]); + new_changelog.write(chop_changes + '\n\n'); + if os.access(changelog_filename, os.R_OK) != 0: + changelog = utils.open_file(changelog_filename, 'r'); + new_changelog.write(changelog.read()); + new_changelog.close(); + if os.access(changelog_filename, os.R_OK) != 0: + os.unlink(changelog_filename); + utils.move(new_changelog_filename, changelog_filename); + + install_count = install_count + 1; + + if not Cnf["Dinstall::Options::No-Mail"]: + mail_message = """Return-Path: %s +From: %s +To: %s +Bcc: troup@auric.debian.org +Subject: %s INSTALLED into stable + +%s +Installing: +%s + +%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, summary, installed_footer) utils.send_mail (mail_message, "") announce (short_summary, 1) @@ -807,6 +915,8 @@ Subject: %s REJECTED def acknowledge_new (changes_filename, summary): global new_ack_new; + changes_filename = os.path.basename(changes_filename); + new_ack_new[changes_filename] = 1; if new_ack_old.has_key(changes_filename): @@ -944,6 +1054,14 @@ def process_it (changes_file): reprocess = 1; orig_tar_id = None; + # Absolutize the filename to avoid the requirement of being in the + # same directory as the .changes file. + changes_file = os.path.abspath(changes_file); + + # And since handling of installs to stable munges with the CWD; + # save and restore it. + cwd = os.getcwd(); + check_signature (changes_file); check_changes (changes_file); while reprocess: @@ -954,6 +1072,9 @@ def process_it (changes_file): action(changes_file); + # Restore CWD + os.chdir(cwd); + ############################################################################### def main(): diff --git a/katie.conf-non-US b/katie.conf-non-US index 2114eee5..d2cf41e9 100644 --- a/katie.conf-non-US +++ b/katie.conf-non-US @@ -135,6 +135,7 @@ Suite OverrideCodeName "potato"; Priority "1"; Untouchable "1"; + ChangeLogBase "dists/stable/non-US/"; }; Proposed-Updates diff --git a/tea b/tea new file mode 100755 index 00000000..54137c58 --- /dev/null +++ b/tea @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# 'Fix' stable to make debian-cd and dpkg -BORGiE users happy +# Copyright (C) 2000 James Troup +# $Id: tea,v 1.1 2000-12-05 04:27:48 troup Exp $ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# And, lo, a great and menacing voice rose from the depths, and with +# great wrath and vehemence it's voice boomed across the +# land... ``hehehehehehe... that *tickles*'' +# -- aj on IRC + +################################################################################ + +import pg, sys, os +import utils, db_access +import apt_pkg; + +################################################################################ + +Cnf = None; +projectB = None; + +################################################################################ + +def main (): + global Cnf, projectB; + + apt_pkg.init(); + + Cnf = apt_pkg.newConfiguration(); + apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); + + Arguments = [('d',"debug","Claire::Options::Debug", "IntVal"), + ('h',"help","Claire::Options::Help"), + ('v',"version","Claire::Options::Version")]; + + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + + projectB = pg.connect('projectb', 'localhost'); + + db_access.init(Cnf, projectB); + + q = projectB.query("SELECT l.path, f.filename FROM files f, location l WHERE f.location = l.id") + ql = q.getresult(); + + for i in ql: + filename = i[0] + i[1]; + if os.access(filename, os.R_OK) == 0: + print filename + +####################################################################################### + +if __name__ == '__main__': + main() + diff --git a/utils.py b/utils.py index 409113ef..6326806f 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,6 @@ # Utility functions # Copyright (C) 2000 James Troup -# $Id: utils.py,v 1.6 2000-12-01 17:33:29 troup Exp $ +# $Id: utils.py,v 1.7 2000-12-05 04:27:48 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -115,7 +115,7 @@ def build_file_list(changes, dsc): # What a mess. FIXME if string.find(section, '/') != -1: component = string.split(section, '/')[0] - if string.lower(component) == "non-us" and string.count(section, '/') > 1: + if string.lower(component) == "non-us" and string.count(section, '/') > 0: s = string.split(section, '/')[1] if s == "main" or s == "non-free" or s == "contrib": # Avoid e.g. non-US/libs component = string.split(section, '/')[0]+ '/' + string.split(section, '/')[1] @@ -196,7 +196,7 @@ def poolify (source, component): ###################################################################################### def move (src, dest): - if os.path.exists(dest) and stat.S_ISDIR(os.stat(dest)[stat.ST_MODE]): + if os.path.exists(dest) and os.path.isdir(dest): dest_dir = dest; else: dest_dir = os.path.dirname(dest); @@ -206,13 +206,13 @@ def move (src, dest): os.umask(umask); #print "Moving %s to %s..." % (src, dest); shutil.copy2(src, dest); - if os.path.exists(dest) and stat.S_ISDIR(os.stat(dest)[stat.ST_MODE]): + if os.path.exists(dest) and os.path.isdir(dest): dest = dest + '/' + os.path.basename(src); os.chmod(dest, 0664); os.unlink(src); def copy (src, dest): - if os.path.exists(dest) and stat.S_ISDIR(os.stat(dest)[stat.ST_MODE]): + if os.path.exists(dest) and os.path.isdir(dest): dest_dir = dest; else: dest_dir = os.path.dirname(dest); @@ -222,7 +222,7 @@ def copy (src, dest): os.umask(umask); #print "Copying %s to %s..." % (src, dest); shutil.copy2(src, dest); - if os.path.exists(dest) and stat.S_ISDIR(os.stat(dest)[stat.ST_MODE]): + if os.path.exists(dest) and os.path.isdir(dest): dest = dest + '/' + os.path.basename(src); os.chmod(dest, 0664); -- GitLab