"""General purpose package removal code for ftpmaster @contact: Debian FTP Master @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup @copyright: 2010 Alexander Reichle-Schmehl @copyright: 2015 Niels Thykier @license: GNU General Public License version 2 or later """ # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup # Copyright (C) 2010 Alexander Reichle-Schmehl # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ################################################################################ # TODO: Insert "random dak quote" here ################################################################################ import commands import apt_pkg from re import sub from daklib.dbconn import * from daklib import utils from daklib.regexes import re_bin_only_nmu import debianbts as bts ################################################################################ def remove(session, reason, suites, removals, whoami=None, partial=False, components=None, done_bugs=None, date=None, carbon_copy=None, close_related_bugs=False): """Batch remove a number of packages Verify that the files listed in the Files field of the .dsc are those expected given the announced Format. @type session: SQLA Session @param session: The database session in use @type reason: string @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by )") @type suites: list @param suites: A list of the suite names in which the removal should occur @type removals: list @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following for 4 items from the database (in order): package, version, architecture, (database) id. For source packages, the "architecture" should be set to "source". @type partial: bool @param partial: Whether the removal is "partial" (e.g. architecture specific). @type components: list @param components: List of components involved in a partial removal. Can be an empty list to not restrict the removal to any components. @type whoami: string @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami() @type date: string @param date: The date of the removal. Defaults to commands.getoutput("date -R") @type done_bugs: list @param done_bugs: A list of bugs to be closed when doing this removal. @type close_related_bugs: bool @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented for more than one suite. @type carbon_copy: list @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike "dak rm". @rtype: None @return: Nothing """ # Generate the summary of what's to be removed d = {} summary = "" sources = [] binaries = [] whitelists = [] versions = [] suite_ids_list = [] suites_list = utils.join_with_commas_and(suites) cnf = utils.get_conf() con_components = None ####################################################################################################### if not reason: raise ValueError("Empty removal reason not permitted") if not removals: raise ValueError("Nothing to remove!?") if not suites: raise ValueError("Removals without a suite!?") if whoami is None: whoami = utils.whoami() if date is None: date = commands.getoutput("date -R") if partial: component_ids_list = [] for componentname in components: component = get_component(componentname, session=session) if component is None: raise ValueError("component '%s' not recognised." % componentname) else: component_ids_list.append(component.component_id) con_components = "AND component IN (%s)" % ", ".join([str(i) for i in component_ids_list]) for i in removals: package = i[0] version = i[1] architecture = i[2] if package not in d: d[package] = {} if version not in d[package]: d[package][version] = [] if architecture not in d[package][version]: d[package][version].append(architecture) for package in sorted(removals): versions = sorted(d[package], cmp=apt_pkg.version_compare) for version in versions: d[package][version].sort(utils.arch_compare_sw) summary += "%10s | %10s | %s\n" % (package, version, ", ".join(d[package][version])) for package in summary.split("\n"): for row in package.split("\n"): element = row.split("|") if len(element) == 3: if element[2].find("source") > 0: sources.append("%s_%s" % tuple(elem.strip(" ") for elem in element[:2])) element[2] = sub("source\s?,?", "", element[2]).strip(" ") if element[2]: binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element)) dsc_type_id = get_override_type('dsc', session).overridetype_id deb_type_id = get_override_type('deb', session).overridetype_id for suite in suites: s = get_suite(suite, session=session) if s is not None: suite_ids_list.append(s.suite_id) whitelists.append(s.mail_whitelist) ####################################################################################################### log_filename = cnf["Rm::LogFile"] log822_filename = cnf["Rm::LogFile822"] with utils.open_file(log_filename, "a") as logfile, utils.open_file(log822_filename, "a") as logfile822: logfile.write("=========================================================================\n") logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) if done_bugs: logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs))) logfile.write("\n------------------- Reason -------------------\n%s\n" % reason) logfile.write("----------------------------------------------\n") logfile822.write("Date: %s\n" % date) logfile822.write("Ftpmaster: %s\n" % whoami) logfile822.write("Suite: %s\n" % suites_list) if sources: logfile822.write("Sources:\n") for source in sources: logfile822.write(" %s\n" % source) if binaries: logfile822.write("Binaries:\n") for binary in binaries: logfile822.write(" %s\n" % binary) logfile822.write("Reason: %s\n" % reason.replace('\n', '\n ')) if done_bugs: logfile822.write("Bug: %s\n" % (", ".join(done_bugs))) for i in removals: package = i[0] architecture = i[2] package_id = i[3] for suite_id in suite_ids_list: if architecture == "source": session.execute("DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid", {'packageid': package_id, 'suiteid': suite_id}) else: session.execute("DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid", {'packageid': package_id, 'suiteid': suite_id}) # Delete from the override file if partial: if architecture == "source": type_id = dsc_type_id else: type_id = deb_type_id # TODO: Fix this properly to remove the remaining non-bind argument session.execute("DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), {'package': package, 'typeid': type_id, 'suiteid': suite_id}) session.commit() # ### REMOVAL COMPLETE - send mail time ### # # If we don't have a Bug server configured, we're done if "Dinstall::BugServer" not in cnf: if done_bugs or close_related_bugs: utils.warn("Cannot send mail to BugServer as Dinstall::BugServer is not configured") logfile.write("=========================================================================\n") logfile822.write("\n") return # read common subst variables for all bug closure mails Subst_common = {} Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] Subst_common["__CC__"] = "X-DAK: dak rm" if carbon_copy: Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy) Subst_common["__SUITE_LIST__"] = suites_list Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % (suites_list) Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"] Subst_common["__WHOAMI__"] = whoami # Send the bug closing messages if done_bugs: Subst_close_rm = Subst_common bcc = [] if cnf.find("Dinstall::Bcc") != "": bcc.append(cnf["Dinstall::Bcc"]) if cnf.find("Rm::Bcc") != "": bcc.append(cnf["Rm::Bcc"]) if bcc: Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc) else: Subst_close_rm["__BCC__"] = "X-Filler: 42" summarymail = "%s\n------------------- Reason -------------------\n%s\n" % (summary, reason) summarymail += "----------------------------------------------\n" Subst_close_rm["__SUMMARY__"] = summarymail for bug in done_bugs: Subst_close_rm["__BUG_NUMBER__"] = bug if close_related_bugs: mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close-with-related") else: mail_message = utils.TemplateSubst(Subst_close_rm,cnf["Dir::Templates"]+"/rm.bug-close") utils.send_mail(mail_message, whitelists=whitelists) # close associated bug reports if close_related_bugs: Subst_close_other = Subst_common bcc = [] wnpp = utils.parse_wnpp_bug_file() versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) if len(versions) == 1: Subst_close_other["__VERSION__"] = versions[0] else: logfile.write("=========================================================================\n") logfile822.write("\n") raise ValueError("Closing bugs with multiple package versions is not supported. Do it yourself.") if bcc: Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) else: Subst_close_other["__BCC__"] = "X-Filler: 42" # at this point, I just assume, that the first closed bug gives # some useful information on why the package got removed Subst_close_other["__BUG_NUMBER__"] = done_bugs[0] if len(sources) == 1: source_pkg = source.split("_", 1)[0] else: logfile.write("=========================================================================\n") logfile822.write("\n") raise ValueError("Closing bugs for multiple source packages is not supported. Please do it yourself.") Subst_close_other["__BUG_NUMBER_ALSO__"] = "" Subst_close_other["__SOURCE__"] = source_pkg merged_bugs = set() other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded') if other_bugs: for bugno in other_bugs: if bugno not in merged_bugs: for bug in bts.get_status(bugno): for merged in bug.mergedwith: other_bugs.remove(merged) merged_bugs.add(merged) logfile.write("Also closing bug(s):") logfile822.write("Also-Bugs:") for bug in other_bugs: Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," logfile.write(" " + str(bug)) logfile822.write(" " + str(bug)) logfile.write("\n") logfile822.write("\n") if source_pkg in wnpp: logfile.write("Also closing WNPP bug(s):") logfile822.write("Also-WNPP:") for bug in wnpp[source_pkg]: # the wnpp-rm file we parse also contains our removal # bugs, filtering that out if bug != Subst_close_other["__BUG_NUMBER__"]: Subst_close_other["__BUG_NUMBER_ALSO__"] += str(bug) + "-done@" + cnf["Dinstall::BugServer"] + "," logfile.write(" " + str(bug)) logfile822.write(" " + str(bug)) logfile.write("\n") logfile822.write("\n") mail_message = utils.TemplateSubst(Subst_close_other, cnf["Dir::Templates"]+"/rm.bug-close-related") if Subst_close_other["__BUG_NUMBER_ALSO__"]: utils.send_mail(mail_message) logfile.write("=========================================================================\n") logfile822.write("\n")