Skip to content

D
Dak

喜羊羊3508 / Dak
大约 1 年 前同步成功

通知 1
Star 0
Fork 0
D
Dak

体验新版 GitCode，发现更多精彩内容 >>

提交 15449de7 编写于 作者: J Joerg Jaspert
浏览文件
操作

And away with the old scripts

上级 9639afdb
隐藏空白更改
内联 并排
Showing with 0 addition and 669 deletion
config/debian/cron.daily 已删除 100755 → 0
浏览文件 @ 9639afdb
#! /bin/bash
#
# Run daily via cron, out of dak's crontab.
set -e
set -o pipefail
set -u
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
# common functions are "outsourced"
. "${configdir}/common"
################################################################################
TMPFILE=$( mktemp -p ${TMPDIR} )
TMPCNTB=$( mktemp -p ${TMPDIR} )
function cleanup {
ERRVAL=$?
rm -f ${TMPFILE} ${TMPCNTB}
exit ${ERRVAL}
}
trap cleanup SIGHUP SIGINT SIGPIPE SIGTERM EXIT ERR
# log to dinstall's logfile instead of sending email
PROGRAM="cron.daily"
# Start logging
NOW=`date "+%Y.%m.%d-%H:%M:%S"`
LOGFILE="$logdir/daily_${NOW}.log"
exec >> "$LOGFILE" 2>&1
# get the latest list of wnpp bugs and their source packages
wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
chmod go+r ${TMPFILE}
mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
# Push files over to security
# The key over there should have the following set for the ssh key:
# command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity"
psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \
xz -3 | \
ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
-i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
# Update wanna-build dump
log "Update wanna-build database dump"
$base/dak/scripts/nfu/get-w-b-db
reports
clean_debbugs
# Generate list of override disparities
dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
# Generate stats about the new queue
dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
# Generate the contributor data
# FIXME: In a day or three, when this worked from cron without
# failure, redirect its output to dev/null. Alternatively until then
# enrico added a --quiet and we use that.
log "Submitting data to contributors"
REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
# Post with curl as a workaround for #801506
# See https://wiki.debian.org/ServicesSSL#curl
dir=/etc/ssl/ca-debian
test -d $dir && capath="--capath $dir"
curl -s $capath https://contributors.debian.org/contributors/post \
-F source=ftp.debian.org \
-F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \
-F data=@${TMPCNTB} > ${TMPCNTB}.result
cat ${TMPCNTB}.result
rm -f ${TMPCNTB}.result
${scriptsdir}/link_morgue.sh
################################################################################
log "Finally, all is done, compressing logfile"
exec > /dev/null 2>&1
bzip2 -9 "$LOGFILE"
config/debian/cron.dinstall 已删除 100755 → 0
浏览文件 @ 9639afdb
#!/bin/bash
# No way I try to deal with a crippled sh just for POSIX foo.
# Copyright (C) 2009-2015 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# Homer: Are you saying you're never going to eat any animal again? What
# about bacon?
# Lisa: No.
# Homer: Ham?
# Lisa: No.
# Homer: Pork chops?
# Lisa: Dad, those all come from the same animal.
# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
# exit on errors
set -e
set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the functions is
# the important part here)
set -E
# import the general variable set.
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
########################################################################
# Functions #
########################################################################
# common functions are "outsourced"
. "${configdir}/common"
# source the dinstall functions
. "${configdir}/dinstall.functions"
########################################################################
########################################################################
# Function to save which stage we are in, so we can restart an interrupted
# dinstall. Or even run actions in parallel, if we dare to, by simply
# backgrounding the call to this function. But that should only really be
# done for things we don't care much about.
#
# This should be called with the first argument being an array, with the
# members
# - FUNC - the function name to call
# - ARGS - Possible arguments to hand to the function. Can be the empty string
# - TIME - The timestamp name. Can be the empty string
# - ERR - if this is the string false, then the call will be surrounded by
# set +e ... set -e calls, so errors in the function do not exit
# dinstall. Can be the empty string, meaning true.
#
# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
# ADDED FOR DINSTALL FEATURES!
function stage() {
ARGS='GO[@]'
local "${!ARGS}"
local error=${ERR:-"true"}
ARGS=${ARGS:-""}
log "########## DINSTALL BEGIN: ${FUNC} ${ARGS} ##########"
local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
STAGEFILE=${STAGEFILE// /_}
if [ -f "${STAGEFILE}" ]; then
local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
local unixtime=$(date +%s)
local difference=$(( $unixtime - $stamptime ))
if [ ${difference} -ge 14400 ]; then
log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
else
log "Did already run ${FUNC}, not calling again..."
fi
return
fi
debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
# Make sure we are always at the same place. If a function wants
# to be elsewhere, it has to cd first!
cd ${configdir}
# Now redirect the output into $STAGEFILE.log. In case it errors
# out somewhere our errorhandler trap can then mail the contents
# of $STAGEFILE.log only, instead of a whole dinstall logfile.
# Short error mails ftw!
exec >> "${STAGEFILE}.log" 2>&1
if [ -f "${LOCK_STOP}" ]; then
log "${LOCK_STOP} exists, exiting immediately"
exit 42
fi
# Do we care about trouble in the function we call?
if [ "${error}" = "false" ]; then
set +e
fi
${FUNC} ${ARGS}
# No matter what happened in the function, we make sure we have
# set -e default state back
set -e
# Make sure we are always at the same place.
cd ${configdir}
# We always use the same umask. If a function wants to do
# different, fine, but we reset.
umask 022
touch "${STAGEFILE}"
if [ -n "${TIME}" ]; then
ts "${TIME}"
fi
# And the output goes back to the normal logfile
exec >> "$LOGFILE" 2>&1
# Now we should make sure that we have a usable dinstall.log, so
# append the $STAGEFILE.log to it.
cat "${STAGEFILE}.log" >> "${LOGFILE}"
rm -f "${STAGEFILE}.log"
log "########## DINSTALL END: ${FUNC} ##########"
if [ -f "${LOCK_STOP}" ]; then
log "${LOCK_STOP} exists, exiting immediately"
exit 42
fi
}
########################################################################
# We need logs.
LOGFILE="$logdir/dinstall.log"
exec >> "$LOGFILE" 2>&1
# And now source our default config
. "${configdir}/dinstall.variables"
# Make sure we start out with a sane umask setting
umask 022
# And use one locale, no matter what the caller has set
export LANG=C
export LC_ALL=C
touch "${DINSTALLSTART}"
ts "startup"
DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
state "Startup"
lockfile -l 3600 "${LOCK_DAILY}"
trap onerror ERR
trap remove_daily_lock EXIT TERM HUP INT QUIT
touch "${LOCK_BRITNEY}"
# This loop simply wants to be fed by a list of values (see below)
# made out of 5 columns.
# The first four are the array values for the stage function, the
# fifth tells us if we should background the stage call.
#
# - FUNC - the function name to call
# - ARGS - Possible arguments to hand to the function. Can be the empty string
# - TIME - The timestamp name. Can be the empty string
# - ERR - if this is the string false, then the call will be surrounded by
# set +e ... set -e calls, so errors in the function do not exit
# dinstall. Can be the empty string, meaning true.
# - BG - Background the function stage?
#
# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
#
# NOTE 1: There are two special values for the first column (FUNC).
# STATE - do not call stage function, call the state
# function to update the public statefile "where is dinstall"
# NOSTAGE - do not call stage function, call the command directly.
#
# Note 2: If you want to hand an empty value to the stage function,
# use the word "none" in the list below.
while read FUNC ARGS TIME ERR BACKGROUND; do
debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"
# Empty values in the value list are the string "none" (or the
# while read loop won't work). Here we ensure that variables that
# can be empty, are empty if the string none is set for them.
for var in ARGS TIME; do
if [[ ${!var} == none ]]; then
typeset ${var}=''
fi
done
# ERR/BACKGROUND are boolean, check that they are.
for var in ERR BACKGROUND; do
if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then
error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}"
fi
done
case ${FUNC} in
STATE)
state ${ARGS}
;;
NOSTAGE)
${ARGS}
;;
*)
GO=(
FUNC=${FUNC}
TIME=${TIME}
ARGS=${ARGS}
ERR=${ERR}
)
if [[ ${BACKGROUND} == true ]]; then
stage $GO &
else
stage $GO
fi
;;
esac
done < <(cat - <<EOF
savetimestamp none none false false
qa1 none init true true
pg_timestamp predinstall pg_dump1 false false
updates none External\ Updates false false
i18n1 none i18n\ 1 false false
dep11 none dep11\ 1 false false
NOSTAGE lockaccepted none false false
punew stable-new p-u-new false false
opunew oldstable-new o-p-u-new false false
backports_policy none backports-policy false false
cruft none cruft false false
STATE indices none false false
dominate none dominate false false
autocruft none autocruft false false
fingerprints none import-keyring false false
overrides none overrides false false
mpfm none pkg-file-mapping false false
STATE packages/contents none false false
packages none apt-ftparchive false false
STATE dists/ none false false
pdiff none pdiff false false
release none release\ files false false
dakcleanup none cleanup false false
STATE scripts none false false
mkmaintainers none mkmaintainers false false
copyoverrides none copyoverrides false false
mklslar none mklslar false false
mkfilesindices none mkfilesindices false false
mkchecksums none mkchecksums false false
signotherfiles none signotherfiles false false
mirror none mirror\ hardlinks false false
NOSTAGE remove_locks none false false
STATE postlock none false false
ddaccess none ddaccessible\ sync true true
changelogs none changelogs false true
pg_timestamp postdinstall pg_dump2 false false
expire none expire_dumps false true
transitionsclean none transitionsclean false true
dm none none false true
bts none none false true
mirrorpush none mirrorpush false true
mirrorpush-backports none mirrorpush-backports false true
i18n2 none i18n\ 2 false true
stats none stats false true
testingsourcelist none none false true
NOSTAGE rm\ -f\ "\${LOCK_BRITNEY}" none false false
cleantransactions none none false false
EOF
)
# FUNC ARGS TIME ERR BG
# we need to wait for the background processes before the end of dinstall
wait
log "Daily cron scripts successful, all done"
exec > "$logdir/afterdinstall.log" 2>&1
if [ -f "${dbdir}/dinstallstart" ]; then
NOW=$(cat "${dbdir}/dinstallstart")
mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
logstats "$logdir/dinstall_${NOW}.log"
bzip2 -9 "$logdir/dinstall_${NOW}.log"
else
error "Problem, I don't know when dinstall started, unable to do log statistics."
NOW=`date "+%Y.%m.%d-%H:%M:%S"`
mv "$LOGFILE" "$logdir/dinstall_${NOW}.log"
bzip2 -9 "$logdir/dinstall_${NOW}.log"
fi
state "all done"
# Now, at the very (successful) end of dinstall, make sure we remove
# our stage files, so the next dinstall run will do it all again.
rm -f ${stagedir}/*
touch "${DINSTALLEND}"
config/debian/cron.hourly 已删除 100755 → 0
浏览文件 @ 9639afdb
#! /bin/bash
#
# Executed hourly via cron, out of dak's crontab.
# Only one of me should ever run.
[ "${FLOCKER}" != "$0" ] && exec env FLOCKER="$0" flock -E 0 -en "$0" "$0"
"$@" || :
set -e
set -o pipefail
set -u
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
PROGRAM="Hourly"
########################################################################
# Functions #
########################################################################
# common functions are "outsourced"
. "${configdir}/common"
dak import-users-from-passwd
# do not run show-new and other stuff in parallel
LOCKFILE="$lockdir/unchecked.lock"
cleanup() {
rm -f "${LOCKFILE}"
}
if lockfile -r16 $LOCKFILE 2> /dev/null; then
trap cleanup EXIT
do_new
dak show-new > /dev/null || true
cleanup
trap - EXIT
fi
dak queue-report -n > $webdir/new.html
dak queue-report -n -d backports-new,backports-policy > ${webdir}/backports-new.html
dak queue-report -8 -d new,byhand,stable-new,oldstable-new,backports-new -r $webdir/stat
sudo -u dak-unpriv dak show-deferred -r $webdir/stat > ${webdir}/deferred.html
dak graph -n new,byhand,stable-new,oldstable-new,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates
cd $webdir
cat removals-20*.txt > removals-full.txt
cat removals.txt >> removals-full.txt
cat removals-20*.822 > removals-full.822
cat removals.822 >> removals-full.822
$base/dak/tools/queue_rss.py -q $queuedir/new -o $webdir/rss/ -d $base/misc -l $base/log/
$base/dak/tools/removals.pl $configdir/removalsrss.rc > $webdir/rss/removals.rss
# cd $masterdir
# timeout -k 180 120 /usr/bin/epydoc -q --html --graph all --css blue -n DAK -o $webdir/epydoc --include-log --graph all --show-imports $masterdir/dak/ $masterdir/dakweb/ || true
# Tell the public mirror to sync its tree
${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
$scriptsdir/generate-d-i
# Update backports ACL
sudo -u dak-unpriv cat /srv/backports-master.debian.org/etc/acl \
| dak acl set-fingerprints backports \
| mail -a "X-Debian: DAK" -e -s "backports: acl changes" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
# do the buildd key updates
BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)
exec >> "${BUILDDFUN}" 2>&1
${scriptsdir}/buildd-remove-keys
${scriptsdir}/buildd-add-keys
${scriptsdir}/buildd-prepare-dir
for keyring in $(dak admin k list-binary); do
dak import-keyring --generate-users "%s" ${keyring}
done
exec >>/dev/null 2>&1
DATE=$(date -Is)
cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" buildd-keys@ftp-master.debian.org
rm -f "${BUILDDFUN}"
config/debian/cron.monthly 已删除 100755 → 0
浏览文件 @ 9639afdb
#! /bin/bash
#
# Run at the beginning of the month via cron, out of dak's crontab.
set -e
set -o pipefail
set -u
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
################################################################################
DATE=`date -d yesterday +%y%m`
DATE=$(date +%Y-%m)
OLDDATE=$(date -d yesterday +%Y-%m)
cd ${base}/log
touch $DATE
ln -sf $DATE current
chmod g+w $DATE
chown dak:ftpteam $DATE
xz -9 ${OLDDATE}
cd /srv/upload.debian.org/queued
sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued -k || true
mv run/log log.${OLDDATE}
sudo -u dak-unpriv touch run/log
xz -9 log.${OLDDATE}
sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued 2>/dev/null
################################################################################
config/debian/cron.unchecked 已删除 100755 → 0
浏览文件 @ 9639afdb
#!/bin/bash
# No way I try to deal with a crippled sh just for POSIX foo.
# Copyright (C) 2009 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# exit on errors
set -e
set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the functions is
# the important part here)
set -E
# import the general variable set.
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
LOCKDAILY=""
LOCKFILE="$lockdir/unchecked.lock"
NOTICE="$lockdir/daily.lock"
LOCK_BUILDD="$lockdir/buildd.lock"
# our name
PROGRAM="unchecked"
if [ -e $NOTICE ]; then
exit 0;
fi
########################################################################
# Functions #
########################################################################
# common functions are "outsourced"
. "${configdir}/common"
STAMP=$(date "+%Y%m%d%H%M")
cleanup() {
rm -f "$LOCKFILE"
if [ ! -z "$LOCKDAILY" ]; then
rm -f "$NOTICE"
fi
}
function do_buildd () {
if lockfile -r3 $NOTICE; then
LOCKDAILY="YES"
make_buildd_dir
wbtrigger
fi
}
########################################################################
# the actual unchecked functions follow #
########################################################################
# And use one locale, no matter what the caller has set
export LANG=C
export LC_ALL=C
# only run one cron.unchecked
if ! lockfile -r8 $LOCKFILE 2> /dev/null; then
# echo "aborting cron.unchecked because $LOCKFILE has already been locked"
exit 0
fi
trap cleanup 0
pg_timestamp preunchecked >/dev/null
# Process policy queues
punew stable-new
opunew oldstable-new
backports_policy
dak clean-suites -a backports-policy,policy
# Finally deal with unchecked
do_unchecked
if [ ! -z "$changes" ]; then
sync_debbugs
do_buildd
fi
dak contents -l 10000 scan-binary
dak contents -l 1000 scan-source
pg_timestamp postunchecked >/dev/null
config/debian/cron.weekly 已删除 100755 → 0
浏览文件 @ 9639afdb
#!/bin/bash
#
# Run once a week via cron, out of dak's crontab.
set -e
set -o pipefail
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the functions is
# the important part here)
set -E
export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
. $SCRIPTVARS
# Start logging
NOW=`date "+%Y.%m.%d-%H:%M:%S"`
LOGFILE="$logdir/weekly_${NOW}.log"
exec > "$LOGFILE" 2>&1
cleanup() {
echo "Cleanup"
rm -f "$LOGFILE"
}
trap cleanup 0
################################################################################
# Purge empty directories
echo "Purging empty directories in $ftpdir/pool/"
if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then
find $ftpdir/pool/ -type d -empty | xargs rmdir;
fi
# Do git cleanup stuff
echo "Doing git stuff"
cd /srv/ftp.debian.org/git/dak.git
git gc --prune
git update-server-info
# now workaround a git bug not honoring the setup in logs/*
# (fix in development, but until it reached backports.org.......)
chmod -R g+w logs/
echo "Fixing symlinks in $ftpdir"
symlinks -d -r $ftpdir
echo "Finally, all is done, compressing logfile"
exec > /dev/null 2>&1
bzip2 -9 "$LOGFILE"
################################################################################
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或