diff --git a/contrib/job-scripts/coreudmp-handling/README b/contrib/job-scripts/coreudmp-handling/README new file mode 100644 index 0000000000000000000000000000000000000000..a04c6a60a48557c74c81d401ad0de62d1b950a98 --- /dev/null +++ b/contrib/job-scripts/coreudmp-handling/README @@ -0,0 +1,33 @@ +These are a set of scripts that will: + +* Setup the core pattern to the "coredumps" sub directory of the + current job and sets the core file size limit to "unlimited" +* Restore the original settings once the job finishes running tests +* Links the core files (if any) on the job level "coredumps" subdir + into the test result specific "coredumps" dir based on the time + stamps of the test directories and the coredump files. + +If your system lacks the prlimit utility, then you have to set core +file size limits yourself. Options to do so include runnning: + + $ ulimit -c unlimited + +Or setting it permanently with an entry in /etc/security/limits.conf, +or adding a file to /etc/security/limits.d. + +To enable these scripts permanently, copy them to the Avocado +systemwide job scripts directories, located by default at +/etc/avocado/scripts/job, respecting the pre and post directories. + +To enable these scripts during a development session, set the +following configuration content: + +[plugins.jobscripts] +pre = /pre.d/ +post = /post.d/ +warn_non_existing_dir = True +warn_non_zero_status = True + +WARNING: Since the core pattern is a system wide setting, if you run +multiple parallel jobs with these scripts, the result is pretty much +unpredictable and prone to errors. diff --git a/contrib/job-scripts/coreudmp-handling/post.d/001-restore-core-pattern b/contrib/job-scripts/coreudmp-handling/post.d/001-restore-core-pattern new file mode 100755 index 0000000000000000000000000000000000000000..7a8c4c3cba852664ca2156dd123118287eea1d5b --- /dev/null +++ b/contrib/job-scripts/coreudmp-handling/post.d/001-restore-core-pattern @@ -0,0 +1,28 @@ +#!/bin/sh -e +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2016 +# +# This script restores the original settings of the systemwide kernel +# core pattern once the job finishes running tests +# + +if [ -z $AVOCADO_JOB_LOGDIR ]; then + exit 1 +fi + +BACKUP_CORE_PATTERN="$AVOCADO_JOB_LOGDIR/coredumps/.backup_core_pattern" +if [ -f $BACKUP_CORE_PATTERN ]; then + cat $BACKUP_CORE_PATTERN > /proc/sys/kernel/core_pattern + rm $BACKUP_CORE_PATTERN +fi diff --git a/contrib/job-scripts/coreudmp-handling/post.d/002-link-core-to-tests b/contrib/job-scripts/coreudmp-handling/post.d/002-link-core-to-tests new file mode 100755 index 0000000000000000000000000000000000000000..398ad842ec59c602c886f0e3e957216e8a71bae5 --- /dev/null +++ b/contrib/job-scripts/coreudmp-handling/post.d/002-link-core-to-tests @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2016 +# +# This script links the core files (if any) on the job level +# "coredumps" subdir into the test result specific "coredumps" dir +# based on the time stamps of the test directories and the timestamps +# of coredump files. +# + +import glob +import os +import sys + +if 'AVOCADO_JOB_LOGDIR' not in os.environ: + sys.exit(1) + +AVOCADO_JOB_LOGDIR = os.environ['AVOCADO_JOB_LOGDIR'] +if not os.path.isdir(AVOCADO_JOB_LOGDIR): + sys.exit(2) + +COREDUMPS_DIR = os.path.join(AVOCADO_JOB_LOGDIR, 'coredumps') +COREDUMPS = glob.glob(os.path.join(COREDUMPS_DIR, 'core.*')) +if not COREDUMPS: + sys.exit(0) + +# If we reached this far, there are core dumps, so let's attempt to +# link them to the test results directories. +# +# This pattern list can be expanded, and if a match occurs, it should +# return as the first group member the PID (an unsigned integer) of +# the process which may have an associated coredump file. + +TESTS_DIR = os.path.join(AVOCADO_JOB_LOGDIR, 'test-results') + + +def symlink_coredumps(): + for test_dir in (os.path.join(TESTS_DIR, _) for _ in os.listdir(TESTS_DIR)): + try: + debug_log = os.path.join(test_dir, "debug.log") + start = os.path.getctime(os.path.join(test_dir, "sysinfo")) + stop = os.path.getmtime(debug_log) + dst_dir = None + for coredump in COREDUMPS: + ctime = os.path.getctime(coredump) + if ctime <= stop and ctime >= start: + if not dst_dir: + dst_dir = os.path.join(test_dir, "coredumps") + os.makedirs(dst_dir) + try: + dst = os.path.join(dst_dir, os.path.basename(coredump)) + os.symlink(os.path.relpath(coredump, dst_dir), dst) + except Exception: + pass + except Exception: + pass + +if __name__ == "__main__": + symlink_coredumps() diff --git a/contrib/job-scripts/coreudmp-handling/pre.d/001-set-core-pattern b/contrib/job-scripts/coreudmp-handling/pre.d/001-set-core-pattern new file mode 100755 index 0000000000000000000000000000000000000000..0132449389730b1746df675c96f4b85aae0e269f --- /dev/null +++ b/contrib/job-scripts/coreudmp-handling/pre.d/001-set-core-pattern @@ -0,0 +1,34 @@ +#!/bin/sh -e +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2016 +# +# This script set up the core pattern to the "coredumps" sub directory +# of the current job and sets the core file size limit to "unlimited" + +if [ -z $AVOCADO_JOB_LOGDIR ]; then + exit 1 +fi + +# Set the core ulimit for Avocado main process +ppid=$$ +prlimit --core=unlimited --pid=$(cat /proc/$ppid/status | grep PPid | cut -f2) + +COREDUMPS_DIR="$AVOCADO_JOB_LOGDIR/coredumps" +mkdir $COREDUMPS_DIR + +# Save a backup of the core pattern setting +cat /proc/sys/kernel/core_pattern > $COREDUMPS_DIR/.backup_core_pattern + +# Set core pattern +echo "$COREDUMPS_DIR/core.%p" > /proc/sys/kernel/core_pattern