Adjust cronscripts to end up with just one

config/debian/common

@@ -23,6 +23,51 @@ function debug () {
     fi
 }
 
+# Function that only cleans tempfiles, but does not exit or otherwise
+# care about any exit status
+function cleantempfiles() {
+    resolvetmpfiles
+    for TEMPFILE in $TMPFILES; do
+        if [ -n "${TEMPFILE}" ] && [ -f "${TEMPFILE}" ]; then
+            rm -f "${TEMPFILE}"
+        elif [ -n "${TEMPFILE}" ] && [ -d "${TEMPFILE}" ]; then
+            if [ "${TEMPFILE}" != "/" ] && [ "${TEMPFILE}" != "/*" ]; then
+                rm -rf "${TEMPFILE}"
+            fi
+        fi
+    done
+    TMPFILES=""
+}
+
+function resolvetmpfiles() {
+    # If you don't understand this better not touch the script
+    for TEMPFILE in $TEMPFILES; do
+        TMPFILES="${TMPFILES} ${!TEMPFILE:-""}"
+    done
+    TEMPFILES=""
+}
+
+# Function cleanup
+# No arguments
+# Cleans up any known tempfile.
+# Just ensure your script sets the variable
+# TEMPFILES to the names of variables of tempfiles
+# Or TMPFILES to the pathes of tempfiles
+function cleanup() {
+    ERRVAL=$?
+    trap - ERR EXIT TERM HUP INT QUIT
+
+    cleantempfiles
+
+    exit $ERRVAL
+}
+TEMPFILES=${TEMPFILES:-""}
+TMPFILES=${TMPFILES:-""}
+
+
+########################################################################
+########################################################################
+
 function wbtrigger() {
     SSHOPT="-n -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240"
     if lockfile -r 3 -l 3600 "${LOCK_BUILDD}"; then
@@ -109,6 +154,11 @@ function do_unchecked () {
     echo "$timestamp": ${changes:-"Nothing to do"}  >> $report
     dak process-upload -a ${UNCHECKED_WITHOUT_LOCK} -d "$unchecked" >> $report
     dak process-commands -d "$unchecked" >> $report
+
+    if [ ! -z "$changes" ]; then
+        sync_debbugs
+        do_buildd
+    fi
 }
 
 # process NEW policy queue
@@ -174,3 +224,131 @@ function get_archiveroot() {
     fi
     echo "${archiveroot}"
 }
+
+# Prepare the trees for buildds, then push wanna-build
+function do_buildd() {
+    if lockfile -r3 $NOTICE; then
+        TEMPFILES="${TEMPFILES} ${NOTICE}"
+        make_buildd_dir
+        wbtrigger
+    fi
+}
+
+# Cleanup policy queues
+function cleanpolicy() {
+    dak clean-suites -a backports-policy,policy
+}
+
+# Scan new packages for contents
+function scancontents() {
+    dak contents -l 10000 scan-binary
+    dak contents -l 1000 scan-source
+}
+
+function ddaccess() {
+    # Tell our dd accessible mirror to sync itself up. Including ftp dir.
+    log "Trigger dd accessible parts sync including ftp dir"
+    ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
+}
+
+
+
+########################################################################
+########################################################################
+########################################################################
+########################################################################
+
+# Function to save which stage we are in, so we can restart an interrupted
+# dinstall. Or even run actions in parallel, if we dare to, by simply
+# backgrounding the call to this function. But that should only really be
+# done for things we don't care much about.
+#
+# This should be called with the first argument being an array, with the
+# members
+#  - FUNC - the function name to call
+#  - ARGS - Possible arguments to hand to the function. Can be the empty string
+#  - TIME - The timestamp name. Can be the empty string
+#  - ERR  - if this is the string false, then the call will be surrounded by
+#           set +e ... set -e calls, so errors in the function do not exit
+#           dinstall. Can be the empty string, meaning true.
+#
+# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
+# ADDED FOR DINSTALL FEATURES!
+function stage() {
+    ARGS='GO[@]'
+    local "${!ARGS}"
+
+    local error=${ERR:-"true"}
+
+    ARGS=${ARGS:-""}
+
+    log "########## ${PROGRAM} BEGIN: ${FUNC} ${ARGS} ##########"
+    local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
+    STAGEFILE=${STAGEFILE// /_}
+    if [ -f "${STAGEFILE}" ]; then
+        local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
+        local unixtime=$(date +%s)
+        local difference=$(( $unixtime - $stamptime ))
+        if [ ${difference} -ge 14400 ]; then
+            log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
+        else
+            log "Did already run ${FUNC}, not calling again..."
+        fi
+        return
+    fi
+
+    debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
+
+    # Make sure we are always at the same place. If a function wants
+    # to be elsewhere, it has to cd first!
+    cd ${configdir}
+
+    # Now redirect the output into $STAGEFILE.log. In case it errors
+    # out somewhere our errorhandler trap can then mail the contents
+    # of $STAGEFILE.log only, instead of a whole ${PROGRAM} logfile.
+    # Short error mails ftw!
+    exec >> "${STAGEFILE}.log" 2>&1
+
+    if [ -f "${LOCK_STOP}" ]; then
+        log "${LOCK_STOP} exists, exiting immediately"
+        exit 42
+    fi
+
+    # Do we care about trouble in the function we call?
+    if [ "${error}" = "false" ]; then
+        set +e
+    fi
+    ${FUNC} ${ARGS}
+
+    # No matter what happened in the function, we make sure we have
+    # set -e default state back
+    set -e
+
+    # Make sure we are always at the same place.
+    cd ${configdir}
+
+    # We always use the same umask. If a function wants to do
+    # different, fine, but we reset.
+    umask 022
+
+    touch "${STAGEFILE}"
+
+    if [ -n "${TIME}" ]; then
+        ts "${TIME}"
+    fi
+
+    # And the output goes back to the normal logfile
+    exec >> "${LOGFILE}" 2>&1
+
+    # Now we should make sure that we have a usable ${PROGRAM}.log, so
+    # append the $STAGEFILE.log to it.
+    cat "${STAGEFILE}.log" >> "${LOGFILE}"
+    rm -f "${STAGEFILE}.log"
+
+    log "########## ${PROGRAM} END: ${FUNC} ##########"
+
+    if [ -f "${LOCK_STOP}" ]; then
+        log "${LOCK_STOP} exists, exiting immediately"
+        exit 42
+    fi
+}

config/debian/cronscript

+#!/bin/bash
+# No way I try to deal with a crippled sh just for POSIX foo.
+
+# Copyright (C) 2009-2015 Joerg Jaspert <joerg@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# Homer: Are you saying you're never going to eat any animal again? What
+#        about bacon?
+# Lisa: No.
+# Homer: Ham?
+# Lisa: No.
+# Homer: Pork chops?
+# Lisa: Dad, those all come from the same animal.
+# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
+
+# exit on errors
+set -e
+set -o pipefail
+# make sure to only use defined variables
+set -u
+# ERR traps should be inherited from functions too. (And command
+# substitutions and subshells and whatnot, but for us the functions is
+# the important part here)
+set -E
+
+# And use one locale, no matter what the caller has set
+export LANG=C
+export LC_ALL=C
+
+ARG=${1:-"meh"}
+# While this check can be done in the following case, some assumptions
+# down there are easier if we sorted out calls without an arg before.
+if [[ ${ARG} == meh ]]; then
+    cat - <<EOF
+This is the FTPMaster cronscript. It needs an argument or it won't do
+anything for you.
+
+Currently accepted Arguments:
+
+   unchecked - Process the unchecked queue
+   dinstall  - Run a dinstall
+   hourly, daily, weekly - Run that part
+
+EOF
+    exit 0
+fi
+
+# Make sure we start out with a sane umask setting
+umask 022
+
+# import the general variable set.
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
+
+# common functions are "outsourced"
+. "${configdir}/common"
+
+# program name is the (lower cased) first argument.
+PROGRAM="${ARG,,}"
+
+# Timestamp when we started
+NOW=$(date "+%Y.%m.%d-%H:%M:%S")
+
+# Which list of tasks should we run?
+TASKLIST="${configdir}/${PROGRAM}.tasks"
+
+# A logfile for every cron script
+LOGFILE="${logdir}/${PROGRAM}_${NOW}.log"
+
+# Each "cronscript" may have a variables and a functions file
+# that we source
+for what in variables functions; do
+    if [[ -f ${configdir}/${PROGRAM}.${what} ]]; then
+        . ${configdir}/${PROGRAM}.${what}
+    fi
+done
+
+# Get rid of tempfiles at the end
+trap cleanup EXIT TERM HUP INT QUIT
+
+case ${ARG,,} in
+    unchecked)
+        # Do not run during dinstall
+        if [[ -e ${LOCK_DAILY} ]]; then
+            exit 0;
+        fi
+        # only run one cron.unchecked and also lock against hourly (newoverview)
+        if ! lockfile -r8 ${LOCK_UNCHECKED} 2> /dev/null; then
+            # log "aborting cron.unchecked because $LOCK_UNCHECKED has already been locked"
+            exit 0
+        fi
+        TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}"
+        ;;
+    dinstall)
+        ;;
+    hourly)
+        # Only one of me should ever run.
+        FLOCKER=${FLOCKER:-""}
+        [  "${FLOCKER}"  != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0"
+        "$@" || :
+        ;;
+    daily)
+        ;;
+    weekly)
+        ;;
+    monthly)
+        ;;
+    *)
+        error "Unknown arg ${ARG}"
+        exit 42
+    ;;
+esac
+
+# An easy access by name for the current log
+ln -sf ${LOGFILE} ${PROGRAM}
+
+# And from here, all output to the log please
+exec >> "$LOGFILE" 2>&1
+
+# The stage function uses this directory
+# This amends the stagedir variable from "vars"
+stagedir="${stagedir}/${PROGRAM}"
+# Ensure the dir exists
+mkdir -p ${stagedir}
+
+# This loop simply wants to be fed by a list of values (see below)
+# made out of 5 columns.
+# The first four are the array values for the stage function, the
+# fifth tells us if we should background the stage call.
+#
+#  - FUNC - the function name to call
+#  - ARGS - Possible arguments to hand to the function. Can be the empty string
+#  - TIME - The timestamp name. Can be the empty string
+#  - ERR  - if this is the string false, then the call will be surrounded by
+#           set +e ... set -e calls, so errors in the function do not exit
+#           dinstall. Can be the empty string, meaning true.
+#  - BG   - Background the function stage?
+#
+# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
+#
+# NOTE 1: There are two special values for the first column (FUNC).
+#         STATE   - do not call stage function, call the state
+#                   function to update the public statefile "where is dinstall"
+#         NOSTAGE - do not call stage function, call the command directly.
+#
+# Note 2: If you want to hand an empty value to the stage function,
+#         use the word "none" in the list below.
+while read FUNC ARGS TIME ERR BACKGROUND; do
+    debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"
+
+    # Empty values in the value list are the string "none" (or the
+    # while read loop won't work). Here we ensure that variables that
+    # can be empty, are empty if the string none is set for them.
+    for var in ARGS TIME; do
+        if [[ ${!var} == none ]]; then
+            typeset ${var}=''
+        fi
+    done
+
+    # ERR/BACKGROUND are boolean, check that they are.
+    for var in ERR BACKGROUND; do
+        if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then
+            error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}"
+        fi
+    done
+
+    case ${FUNC} in
+        STATE)
+            state ${ARGS}
+        ;;
+        NOSTAGE)
+            ${ARGS}
+        ;;
+        *)
+            GO=(
+                FUNC=${FUNC}
+                TIME=${TIME}
+                ARGS=${ARGS}
+                ERR=${ERR}
+            )
+            if [[ ${BACKGROUND} == true ]]; then
+                stage $GO &
+            else
+                stage $GO
+            fi
+        ;;
+    esac
+done < <(grep -v '^#' ${TASKLIST} )
+
+# we need to wait for the background processes before the end of the cron script
+wait
+
+
+# Common to all cron scripts
+log "Cron script successful, all done"
+# Redirect output to another file, as we want to compress our logfile
+# and ensure its no longer used
+exec > "$logdir/after${PROGRAM}.log" 2>&1
+
+case ${ARG,,} in
+    unchecked)
+        ;;
+    dinstall)
+        logstats ${LOGFILE}
+        state "all done"
+        touch "${DINSTALLEND}"
+        ;;
+    hourly)
+        ;;
+    daily)
+        ;;
+    weekly)
+        ;;
+    monthly)
+        ;;
+esac
+
+# Now, at the very (successful) end of this run, make sure we remove
+# our stage files, so the next dinstall run will do it all again.
+rm -f ${stagedir}/*
+bzip2 -9 ${LOGFILE}
+# FIXME: Mail the log when its non-empty
+[[ -s "${logdir}/after${PROGRAM}.log" ]] || rm "${logdir}/after${PROGRAM}.log"

config/debian/daily.functions

+# -*- mode:sh -*-
+
+# get the latest list of wnpp bugs and their source packages
+function wnppbugs() {
+    TMPFILE=$( mktemp -p ${TMPDIR} )
+    TEMPFILES="${TEMPFILES} ${TMPFILE}"
+    wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
+    chmod go+r ${TMPFILE}
+    mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
+}
+
+# Push files over to security
+function pushfilesdb() {
+    # The key over there should have the following set for the ssh key:
+    #  command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity"
+    psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \
+        xz -3 | \
+        ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
+            -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
+}
+
+# Update wanna-build dump
+function wbdump() {
+    log "Update wanna-build database dump"
+    $base/dak/scripts/nfu/get-w-b-db
+}
+
+# Generate list of override disparities
+function overridedisp() {
+    dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
+}
+
+# Generate stats about the new queue
+function newstats() {
+    dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
+}
+
+# Generate the contributor data
+function contributor() {
+    log "Submitting data to contributors"
+    TMPCNTB=$( mktemp -p ${TMPDIR} )
+    TEMPFILES="${TEMPFILES} ${TMPCNTB}"
+    REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
+
+    # Post with curl as a workaround for #801506
+    # See https://wiki.debian.org/ServicesSSL#curl
+    dir=/etc/ssl/ca-debian
+    test -d $dir && capath="--capath $dir"
+    curl -s $capath https://contributors.debian.org/contributors/post \
+         -F source=ftp.debian.org \
+         -F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \
+         -F data=@${TMPCNTB} > ${TMPCNTB}.result
+    cat ${TMPCNTB}.result
+    rm -f ${TMPCNTB}.result
+}
+
+function linkmorgue() {
+    ${scriptsdir}/link_morgue.sh
+}

config/debian/daily.tasks

+# FUNC                 ARGS                       TIME                       ERR     BG
+linkmorgue             none                       none                       false   true
+wnppbugs               none                       none                       false   false
+pushfilesdb            none                       none                       false   false
+wbdump                 none                       none                       false   false
+reports                none                       none                       false   false
+clean_debbugs          none                       none                       false   false
+overridedisp           none                       none                       false   false
+newstats               none                       none                       false   false
+contributor            none                       none                       false   false
+finalize               none                       none                       false   false
+

config/debian/dinstall.functions

@@ -5,6 +5,7 @@ function ts() {
 }
 
 # Remove daily lock
+# FIXME: Dead code when switched to cronscript
 function remove_daily_lock() {
     rm -f $LOCK_DAILY
 }
@@ -50,7 +51,7 @@ function onerror() {
 }
 
 ########################################################################
-# the actual dinstall functions follow                                 #
+# the actual functions follow                                          #
 ########################################################################
 
 # pushing merkels QA user, part one
@@ -482,12 +483,6 @@ function bts() {
     sudo -u dak-unpriv dak bts-categorize
 }
 
-function ddaccess() {
-    # Tell our dd accessible mirror to sync itself up. Including ftp dir.
-    log "Trigger dd accessible parts sync including ftp dir"
-    ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
-}
-
 function mirrorpush() {
     log "Checking the public archive copies..."
 
@@ -615,7 +610,6 @@ function logstats() {
 
 # save timestamp when we start
 function savetimestamp() {
-    NOW=`date "+%Y.%m.%d-%H:%M:%S"`
     echo ${NOW} > "${dbdir}/dinstallstart"
 }
 
@@ -663,6 +657,7 @@ function changelogs() {
     fi
 }
 
+# Generate a list of extra mirror files, sha256sum em and sign that
 function signotherfiles() {
     log "Signing extra mirror files"
 
@@ -681,3 +676,13 @@ function signotherfiles() {
         rm -f ${TMPLO}
     done
 }
+
+function startup() {
+    touch "${DINSTALLSTART}"
+    ts "startup"
+    DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
+    lockfile -l 3600 "${LOCK_DAILY}"
+    trap onerror ERR
+    TEMPFILES="${TEMPFILES} ${LOCK_DAILY}"
+    touch "${LOCK_BRITNEY}"
+}

config/debian/dinstall.tasks

+# FUNC                 ARGS                       TIME                       ERR     BG
+STATE                  startup                    none                       false   false
+NOSTAGE                startup                    none                       false   false
+savetimestamp          none                       none                       false   false
+qa1                    none                       init                       true    true
+pg_timestamp           predinstall                pg_dump1                   false   false
+updates                none                       External\ Updates          false   false
+i18n1                  none                       i18n\ 1                    false   false
+dep11                  none                       dep11\ 1                   false   false
+NOSTAGE                lockaccepted               none                       false   false
+punew                  stable-new                 p-u-new                    false   false
+opunew                 oldstable-new              o-p-u-new                  false   false
+backports_policy       none                       backports-policy           false   false
+cruft                  none                       cruft                      false   false
+STATE                  indices                    none                       false   false
+dominate               none                       dominate                   false   false
+autocruft              none                       autocruft                  false   false
+fingerprints           none                       import-keyring             false   false
+overrides              none                       overrides                  false   false
+mpfm                   none                       pkg-file-mapping           false   false
+STATE                  packages/contents          none                       false   false
+packages               none                       apt-ftparchive             false   false
+STATE                  dists/                     none                       false   false
+pdiff                  none                       pdiff                      false   false
+release                none                       release\ files             false   false
+dakcleanup             none                       cleanup                    false   false
+STATE                  scripts                    none                       false   false
+mkmaintainers          none                       mkmaintainers              false   false
+copyoverrides          none                       copyoverrides              false   false
+mklslar                none                       mklslar                    false   false
+mkfilesindices         none                       mkfilesindices             false   false
+mkchecksums            none                       mkchecksums                false   false
+signotherfiles         none                       signotherfiles             false   false
+mirror                 none                       mirror\ hardlinks          false   false
+NOSTAGE                remove_locks               none                       false   false
+STATE                  postlock                   none                       false   false
+ddaccess               none                       ddaccessible\ sync         true    true
+changelogs             none                       changelogs                 false   true
+pg_timestamp           postdinstall               pg_dump2                   false   false
+expire                 none                       expire_dumps               false   true
+transitionsclean       none                       transitionsclean           false   true
+dm                     none                       none                       false   true
+bts                    none                       none                       false   true
+mirrorpush             none                       mirrorpush                 false   true
+mirrorpush-backports   none                       mirrorpush-backports       false   true
+i18n2                  none                       i18n\ 2                    false   true
+stats                  none                       stats                      false   true
+testingsourcelist      none                       none                       false   true
+NOSTAGE                rm\ -f\ "\${LOCK_BRITNEY}" none                       false   false
+cleantransactions      none                       none                       false   false

config/debian/dinstall.variables

@@ -2,9 +2,6 @@
 # usually we are not using debug logs. Set to 1 if you want them.
 DEBUG=0
 
-# our name
-PROGRAM="dinstall"
-
 # where do we want mails to go? For example log entries made with error()
 if [ "x$(hostname -s)x" != "xfranckx" ]; then
     # Not our ftpmaster host
@@ -22,9 +19,6 @@ DINSTALLSTART="${lockdir}/dinstallstart"
 # Marker for dinstall end
 DINSTALLEND="${lockdir}/dinstallend"
 
-# lock cron.unchecked (it immediately exits when this exists)
-LOCK_DAILY="$lockdir/daily.lock"
-
 # Lock cron.unchecked from doing work
 LOCK_ACCEPTED="$lockdir/unchecked.lock"
 

config/debian/hourly.functions

+# -*- mode:sh -*-
+
+function importusers() {
+    dak import-users-from-passwd
+}
+
+function newoverview() {
+    # do not run show-new and other stuff in parallel
+    if lockfile -r16 $LOCK_UNCHECKED 2> /dev/null; then
+        TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}"
+        do_new
+        dak show-new > /dev/null || true
+    fi
+    cleantempfiles
+}
+
+function queuereport() {
+    dak queue-report -n > ${webdir}/new.html
+    dak queue-report -n -d backports-new,backports-policy > ${webdir}/backports-new.html
+    dak queue-report -8 -d new,byhand,stable-new,oldstable-new,backports-new -r $webdir/stat
+    sudo -u dak-unpriv dak show-deferred -r ${webdir}/stat > ${webdir}/deferred.html
+    dak graph -n new,byhand,stable-new,oldstable-new,deferred -r ${webdir}/stat -i ${webdir}/stat -x $scriptsdir/rrd-release-freeze-dates
+}
+
+function removalstxt() {
+    cd ${webdir}
+    cat removals-20*.txt > removals-full.txt
+    cat removals.txt >> removals-full.txt
+    cat removals-20*.822 > removals-full.822
+    cat removals.822 >> removals-full.822
+}
+
+function rss() {
+    $base/dak/tools/queue_rss.py -q $queuedir/new -o ${webdir}/rss/ -d $base/misc -l $base/log/
+    $base/dak/tools/removals.pl $configdir/removalsrss.rc > ${webdir}/rss/removals.rss
+}
+
+function gen_di() {
+    $scriptsdir/generate-d-i
+}
+
+function backportsacl() {
+    # Update backports ACL
+    sudo -u dak-unpriv cat /srv/backports-master.debian.org/etc/acl \
+        | dak acl set-fingerprints backports \
+        | mail -a "X-Debian: DAK" -e -s "backports: acl changes" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
+}
+
+# do the buildd key updates
+function builddautosigning() {
+    BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)
+    TEMPFILES="${TEMPFILEs} ${BUILDDFUN}"
+    exec >> "${BUILDDFUN}" 2>&1
+    ${scriptsdir}/buildd-remove-keys
+    ${scriptsdir}/buildd-add-keys
+    ${scriptsdir}/buildd-prepare-dir
+
+    for keyring in $(dak admin k list-binary); do
+        dak import-keyring --generate-users "%s" ${keyring}
+    done
+    exec >>/dev/null 2>&1
+
+    DATE=$(date -Is)
+    cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" buildd-keys@ftp-master.debian.org
+
+    rm -f "${BUILDDFUN}"
+}

config/debian/hourly.tasks

+# FUNC                 ARGS                       TIME                       ERR     BG
+importusers            none                       none                       false   false
+newoverview            none                       none                       false   false
+queuereport            none                       none                       false   true
+removalstxt            none                       none                       false   true
+rss                    none                       none                       false   true
+ddaccess               none                       none                       true    true
+gen_di                 none                       none                       false   false
+backportsacl           none                       none                       false   false
+builddautosigning      none                       none                       false   false
\ No newline at end of file

config/debian/monthly.functions

+# -*- mode:sh -*-
+
+function rotatelog() {
+    cd ${base}/log
+    touch ${DATE}
+    ln -sf ${DATE} current
+    chmod g+w ${DATE}
+    chown dak:ftpteam ${DATE}
+    xz -9 ${OLDDATE}
+}
+
+function rotatequeued() {
+    cd /srv/upload.debian.org/queued
+    sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued -k || true
+    mv run/log log.${OLDDATE}
+    sudo -u dak-unpriv touch run/log
+    xz -9 log.${OLDDATE}
+    sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued 2>/dev/null
+}
\ No newline at end of file

config/debian/monthly.tasks

+# FUNC                 ARGS                       TIME                       ERR     BG
+rotatelog              none                       none                       false   true
+rotatequeued           none                       none                       false   true
\ No newline at end of file

config/debian/monthly.variables

+# -*- mode:sh -*-
+DATE=$(date +%Y-%m)
+OLDDATE=$(date -d yesterday +%Y-%m)

config/debian/unchecked.tasks

+# FUNC                 ARGS                       TIME                       ERR     BG
+pg_timestamp           preunchecked               none                       false   false
+punew                  stable-new                 p-u-new                    false   false
+opunew                 oldstable-new              o-p-u-new                  false   false
+backports_policy       none                       backports-policy           false   false
+cleanpolicy            none                       cleanpolicy                false   false
+do_unchecked           none                       unchecked                  false   false
+scancontents           none                       contentsscan               false   false
+pg_timestamp           postunchecked              none                       false   false

config/debian/unchecked.variables

+# -*- mode:sh -*-
+LOCKFILE="$lockdir/unchecked.lock"
+LOCK_BUILDD="$lockdir/buildd.lock"

config/debian/vars

@@ -40,5 +40,9 @@ umask 022
 unset CDPATH
 GZIP='--rsyncable' ; export GZIP
 
+# lock cron.unchecked (it immediately exits when this exists)
+LOCK_DAILY="$lockdir/daily.lock"
+LOCK_UNCHECKED="$lockdir/unchecked.lock"
+
 # Set the database variables
 eval $(dak admin config db-shell)

config/debian/weekly.functions

+# -*- mode:sh -*-
+
+function purgeempty() {
+    log "Purging empty directories in $ftpdir/pool/"
+
+    if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then
+        find $ftpdir/pool/ -type d -empty | xargs rmdir;
+    fi
+}
+
+function gitcleanup() {
+    log "Doing git stuff"
+    cd ${public}/git/dak.git
+    git gc --prune
+    git update-server-info
+    # now workaround a git bug not honoring the setup in logs/*
+    # (fix in development, but until it reached backports.org.......)
+    chmod -R g+w logs/
+}
+
+function symlinks() {
+    log "Fixing symlinks in $ftpdir"
+    symlinks -d -r $ftpdir
+}

config/debian/weekly.tasks

+# FUNC                 ARGS                       TIME                       ERR     BG
+purgeempty             none                       none                       false   false
+gitcleanup             none                       none                       false   false
+symlinks               none                       none                       false   false