Introduce new OOM testing support

The previous OOM testing support would re-run the entire "main" method each iteration, failing a different malloc each time. When a test suite has 'n' allocations, the number of repeats requires is (n * (n + 1) ) / 2. This gets very large, very quickly. This new OOM testing support instead integrates at the virtTestRun level, so each individual test case gets repeated, instead of the entire test suite. This means the values of 'n' are orders of magnitude smaller. The simple usage is $ VIR_TEST_OOM=1 ./qemuxml2argvtest ... 29) QEMU XML-2-ARGV clock-utc ... OK Test OOM for nalloc=36 .................................... OK 30) QEMU XML-2-ARGV clock-localtime ... OK Test OOM for nalloc=36 .................................... OK 31) QEMU XML-2-ARGV clock-france ... OK Test OOM for nalloc=38 ...................................... OK ... the second lines reports how many mallocs have to be failed, and thus how many repeats of the test will be run. If it crashes, then running under valgrind will often show the problem $ VIR_TEST_OOM=1 ../run valgrind ./qemuxml2argvtest When debugging problems it is also helpful to select an individual test case $ VIR_TEST_RANGE=30 VIR_TEST_OOM=1 ../run valgrind ./qemuxml2argvtest When things get really tricky, it is possible to request that just specific allocs are failed. eg to fail allocs 5 -> 12, use $ VIR_TEST_RANGE=30 VIR_TEST_OOM=1:5-12 ../run valgrind ./qemuxml2argvtest In the worse case, you might want to know the stack trace of the alloc which was failed then VIR_TEST_OOM_TRACE can be set. If it is set to 1 then it will only print if it thinks a mistake happened. This is often not reliable, so setting it to 2 will make it print the stack trace for every alloc that is failed. $ VIR_TEST_OOM_TRACE=2 VIR_TEST_RANGE=30 VIR_TEST_OOM=1:5-5 ../run valgrind ./qemuxml2argvtest 30) QEMU XML-2-ARGV clock-localtime ... OK Test OOM for nalloc=36 !virAllocN /home/berrange/src/virt/libvirt/src/util/viralloc.c:180 virHashCreateFull /home/berrange/src/virt/libvirt/src/util/virhash.c:144 virDomainDefParseXML /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:11745 virDomainDefParseNode /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12646 virDomainDefParse /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12590 testCompareXMLToArgvFiles /home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:106 virtTestRun /home/berrange/src/virt/libvirt/tests/testutils.c:250 mymain /home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:418 (discriminator 2) virtTestMain /home/berrange/src/virt/libvirt/tests/testutils.c:750 ?? ??:0 _start ??:? FAILED Signed-off-by: N Daniel P. Berrange <berrange@redhat.com>

Introduce new OOM testing support
The previous OOM testing support would re-run the entire "main" method each iteration, failing a different malloc each time. When a test suite has 'n' allocations, the number of repeats requires is (n * (n + 1) ) / 2. This gets very large, very quickly. This new OOM testing support instead integrates at the virtTestRun level, so each individual test case gets repeated, instead of the entire test suite. This means the values of 'n' are orders of magnitude smaller. The simple usage is $ VIR_TEST_OOM=1 ./qemuxml2argvtest ... 29) QEMU XML-2-ARGV clock-utc ... OK Test OOM for nalloc=36 .................................... OK 30) QEMU XML-2-ARGV clock-localtime ... OK Test OOM for nalloc=36 .................................... OK 31) QEMU XML-2-ARGV clock-france ... OK Test OOM for nalloc=38 ...................................... OK ... the second lines reports how many mallocs have to be failed, and thus how many repeats of the test will be run. If it crashes, then running under valgrind will often show the problem $ VIR_TEST_OOM=1 ../run valgrind ./qemuxml2argvtest When debugging problems it is also helpful to select an individual test case $ VIR_TEST_RANGE=30 VIR_TEST_OOM=1 ../run valgrind ./qemuxml2argvtest When things get really tricky, it is possible to request that just specific allocs are failed. eg to fail allocs 5 -> 12, use $ VIR_TEST_RANGE=30 VIR_TEST_OOM=1:5-12 ../run valgrind ./qemuxml2argvtest In the worse case, you might want to know the stack trace of the alloc which was failed then VIR_TEST_OOM_TRACE can be set. If it is set to 1 then it will only print if it thinks a mistake happened. This is often not reliable, so setting it to 2 will make it print the stack trace for every alloc that is failed. $ VIR_TEST_OOM_TRACE=2 VIR_TEST_RANGE=30 VIR_TEST_OOM=1:5-5 ../run valgrind ./qemuxml2argvtest 30) QEMU XML-2-ARGV clock-localtime ... OK Test OOM for nalloc=36 !virAllocN /home/berrange/src/virt/libvirt/src/util/viralloc.c:180 virHashCreateFull /home/berrange/src/virt/libvirt/src/util/virhash.c:144 virDomainDefParseXML /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:11745 virDomainDefParseNode /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12646 virDomainDefParse /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12590 testCompareXMLToArgvFiles /home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:106 virtTestRun /home/berrange/src/virt/libvirt/tests/testutils.c:250 mymain /home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:418 (discriminator 2) virtTestMain /home/berrange/src/virt/libvirt/tests/testutils.c:750 ?? ??:0 _start ??:? FAILED Signed-off-by: N Daniel P. Berrange <berrange@redhat.com>
590029f6 · Daniel P. Berrange · cb9b3bc2 · 590029f6 · 590029f6 · 590029f6
9 changed file
--- a/cfg.mk
+++ b/cfg.mk
@@ -949,7 +949,7 @@ exclude_file_name_regexp--sc_libvirt_unmarked_diagnostics = \
 exclude_file_name_regexp--sc_po_check = ^(docs/|src/rpc/gendispatch\.pl$$)

 exclude_file_name_regexp--sc_prohibit_VIR_ERR_NO_MEMORY = \
-  ^(include/libvirt/virterror\.h|daemon/dispatch\.c|src/util/virerror\.c)$$
+  ^(include/libvirt/virterror\.h|daemon/dispatch\.c|src/util/virerror\.c|docs/internals/oomtesting\.html\.in)$$

 exclude_file_name_regexp--sc_prohibit_access_xok = ^src/util/virutil\.c$$


--- a/configure.ac
+++ b/configure.ac
@@ -422,6 +422,8 @@ AC_PATH_PROG([OVSVSCTL], [ovs-vsctl], [ovs-vsctl],
 	[/sbin:/usr/sbin:/usr/local/sbin:$PATH])
 AC_PATH_PROG([SCRUB], [scrub], [scrub],
 	[/sbin:/usr/sbin:/usr/local/sbin:$PATH])
+AC_PATH_PROG([ADDR2LINE], [addr2line], [addr2line],
+        [/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:$PATH])

 AC_DEFINE_UNQUOTED([DMIDECODE],["$DMIDECODE"],
        [Location or name of the dmidecode program])
@@ -452,6 +454,8 @@ if test -n "$RMMOD"; then
 fi
 AC_DEFINE_UNQUOTED([SCRUB],["$SCRUB"],
        [Location or name of the scrub program (for wiping algorithms)])
+AC_DEFINE_UNQUOTED([ADDR2LINE],["$ADDR2LINE"],
+        [Location of addr2line program])

 dnl Specific dir for HTML output ?
 AC_ARG_WITH([html-dir], [AS_HELP_STRING([--with-html-dir=path],

--- a/docs/internals/oomtesting.html.in
+++ b/docs/internals/oomtesting.html.in
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <body>
+    <h1>Out of memory testing</h1>
+
+    <ul id="toc"></ul>
+
+
+    <p>
+      This page describes how to use the test suite todo out of memory
+      testing.
+    </p>
+
+    <h2>Building with OOM testing</h2>
+
+    <p>
+      Since OOM testing requires hooking into the malloc APIs, it is
+      not enabled by default. The flag <code>--enable-test-oom</code>
+      must be given to <code>configure</code>. When this is done the
+      libvirt allocation APIs will have some hooks enabled.
+    </p>
+
+    <pre>
+$ ./configure --enable-test-oom
+</pre>
+
+
+    <h2><a name="basicoom">Basic OOM testing support</a></h2>
+
+    <p>
+      The first step in validating OOM usage is to run a test suite
+      with full OOM testing enabled. This is done by setting the
+      <code>VIR_TEST_OOM=1</code> environment variable. The way this
+      works is that it runs the test once normally to "prime" any
+      static memory allocations. Then it runs it once more counting
+      the total number of memory allocations. Then it runs it in a
+      loop failing a different memory allocation each time. For every
+      memory allocation failure triggered, it expects the test case
+      to return an error. OOM testing is quite slow requiring each
+      test case to be executed O(n) times, where 'n' is the total
+      number of memory allocations. This results in a total number
+      of memory allocations of '(n * (n + 1) ) / 2'
+    </p>
+
+    <pre>
+$ VIR_TEST_OOM=1 ./qemuxml2argvtest
+ 1) QEMU XML-2-ARGV minimal                                           ... OK
+    Test OOM for nalloc=42 .......................................... OK
+ 2) QEMU XML-2-ARGV minimal-s390                                      ... OK
+    Test OOM for nalloc=28 ............................ OK
+ 3) QEMU XML-2-ARGV machine-aliases1                                  ... OK
+    Test OOM for nalloc=38 ...................................... OK
+ 4) QEMU XML-2-ARGV machine-aliases2                                  ... OK
+    Test OOM for nalloc=38 ...................................... OK
+ 5) QEMU XML-2-ARGV machine-core-on                                   ... OK
+    Test OOM for nalloc=37 ..................................... OK
+...snip...
+</pre>
+
+    <p>
+      In this output, the first line shows the normal execution and
+      the test number, and the second line shows the total number
+      of memory allocations from that test case.
+    </p>
+
+    <h3><a name="valgrind">Tracking failures with valgrind</a></h3>
+
+    <p>
+      The test suite should obviously *not* crash during OOM testing.
+      If it does crash, then to assist in tracking down the problem
+      it is worth using valgrind and only running a single test case.
+      For example, supposing test case 5 crashed. Then re-run the
+      test with
+    </p>
+
+    <pre>
+$ VIR_TEST_OOM=1 VIR_TEST_RANGE=5 ../run valgrind ./qemuxml2argvtest
+...snip...
+ 5) QEMU XML-2-ARGV machine-core-on                                   ... OK
+    Test OOM for nalloc=37 ..................................... OK
+...snip...
+    </pre>
+
+    <p>
+      Valgrind should report the cause of the crash - for example a
+      double free or use of uninitialized memory or NULL pointer
+      access.
+    </p>
+
+    <h3><a name="stacktraces">Tracking failures with stack traces</a></h3>
+
+    <p>
+      With some really difficult bugs valgrind is not sufficient to
+      identify the cause. In this case, it is useful to identify the
+      precise allocation which was failed, to allow the code path
+      to the error to be traced. The <code>VIR_TEST_OOM</code>
+      env variable can be given a range of memory allocations to
+      test. So if a test case has 150 allocations, it can be told
+      to only test allocation numbers 7-10. The <code>VIR_TEST_OOM_TRACE</code>
+      variable can be used to print out stack traces.
+    </p>
+
+    <pre>
+$ VIR_TEST_OOM_TRACE=2 VIR_TEST_OOM=1:7-10 VIR_TEST_RANGE=5 \
+    ../run valgrind ./qemuxml2argvtest
+ 5) QEMU XML-2-ARGV machine-core-on                                   ... OK
+    Test OOM for nalloc=37 !virAllocN
+/home/berrange/src/virt/libvirt/src/util/viralloc.c:180
+virDomainDefParseXML
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:11786 (discriminator 1)
+virDomainDefParseNode
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12677
+virDomainDefParse
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12621
+testCompareXMLToArgvFiles
+/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:107
+virtTestRun
+/home/berrange/src/virt/libvirt/tests/testutils.c:266
+mymain
+/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:388 (discriminator 2)
+virtTestMain
+/home/berrange/src/virt/libvirt/tests/testutils.c:791
+__libc_start_main
+??:?
+_start
+??:?
+!virAlloc
+/home/berrange/src/virt/libvirt/src/util/viralloc.c:133
+virDomainDiskDefParseXML
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:4790
+virDomainDefParseXML
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:11797
+virDomainDefParseNode
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12677
+virDomainDefParse
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12621
+testCompareXMLToArgvFiles
+/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:107
+virtTestRun
+/home/berrange/src/virt/libvirt/tests/testutils.c:266
+mymain
+/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:388 (discriminator 2)
+virtTestMain
+/home/berrange/src/virt/libvirt/tests/testutils.c:791
+__libc_start_main
+??:?
+_start
+??:?
+!virAllocN
+/home/berrange/src/virt/libvirt/src/util/viralloc.c:180
+virXPathNodeSet
+/home/berrange/src/virt/libvirt/src/util/virxml.c:609
+virDomainDefParseXML
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:11805
+virDomainDefParseNode
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12677
+virDomainDefParse
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12621
+testCompareXMLToArgvFiles
+/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:107
+virtTestRun
+/home/berrange/src/virt/libvirt/tests/testutils.c:266
+mymain
+/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:388 (discriminator 2)
+virtTestMain
+/home/berrange/src/virt/libvirt/tests/testutils.c:791
+__libc_start_main
+??:?
+_start
+??:?
+!virAllocN
+/home/berrange/src/virt/libvirt/src/util/viralloc.c:180
+virDomainDefParseXML
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:11808 (discriminator 1)
+virDomainDefParseNode
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12677
+virDomainDefParse
+/home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12621
+testCompareXMLToArgvFiles
+/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:107
+virtTestRun
+/home/berrange/src/virt/libvirt/tests/testutils.c:266
+mymain
+/home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:388 (discriminator 2)
+virtTestMain
+/home/berrange/src/virt/libvirt/tests/testutils.c:791
+__libc_start_main
+??:?
+_start
+??:?
+    </pre>
+
+    <h3><a name="noncrash">Non-crash related problems</a></h3>
+
+    <p>
+      Not all memory allocation bugs result in code crashing. Sometimes
+      the code will be silently ignoring the allocation failure, resulting
+      in incorrect data being produced. For example the XML parser may
+      mistakenly treat an allocation failure as indicating that an XML
+      attribute was not set in the input document. It is hard to identify
+      these problems from the test suite automatically. For this, the
+      test suites should be run with <code>VIR_TEST_DEBUG=1</code> set
+      and then stderr analysed for any unexpected data. For example,
+      the XML conversion may show an embedded "(null)" literal, or the
+      test suite might complain about missing elements / attributes
+      in the actual vs expected data. These are all signs of bugs in
+      OOM handling. In the future the OOM tests will be enhanced to
+      validate that an error VIR_ERR_NO_MEMORY is returned for each
+      allocation failed, rather than some other error.
+    </p>
+  </body>
+</html>
--- a/docs/sitemap.html.in
+++ b/docs/sitemap.html.in
@@ -342,6 +342,10 @@
                <a href="internals/locking.html">Lock managers</a>
                <span>Use lock managers to protect disk content</span>
              </li>
+              <li>
+                <a href="internals/oomtesting.html">Out of memory testing</a>
+                <span>Simulating OOM conditions in the test suite</span>
+              </li>
            </ul>
          </li>
          <li>

--- a/tests/qemuargv2xmltest.c
+++ b/tests/qemuargv2xmltest.c
@@ -42,7 +42,6 @@ static int testCompareXMLToArgvFiles(const char *xml,
    char *cmd = NULL;
    int ret = -1;
    virDomainDefPtr vmdef = NULL;
-    char *log;

    if (virtTestLoadFile(cmdfile, &cmd) < 0)
        goto fail;
@@ -53,6 +52,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
                                             cmd, NULL, NULL, NULL)))
        goto fail;

+    if (!virtTestOOMActive()) {
+        char *log;
        if ((log = virtTestLogContentAndReset()) == NULL)
            goto fail;
        if ((*log != '\0') != expect_warning) {
@@ -60,6 +61,7 @@ static int testCompareXMLToArgvFiles(const char *xml,
            goto fail;
        }
        VIR_FREE(log);
+    }

    if (!virDomainDefCheckABIStability(vmdef, vmdef)) {
        fprintf(stderr, "ABI stability check failed on %s", xml);

--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -284,7 +284,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
    if (!(vmdef = virDomainDefParseFile(xml, driver.caps, driver.xmlopt,
                                        QEMU_EXPECTED_VIRT_TYPES,
                                        VIR_DOMAIN_XML_INACTIVE))) {
-        if (flags & FLAG_EXPECT_PARSE_ERROR)
+        if (!virtTestOOMActive() &&
+            (flags & FLAG_EXPECT_PARSE_ERROR))
            goto ok;
        goto out;
    }
@@ -357,7 +358,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
                                     migrateFrom, migrateFd, NULL,
                                     VIR_NETDEV_VPORT_PROFILE_OP_NO_OP,
                                     &testCallbacks))) {
-        if (flags & FLAG_EXPECT_FAILURE) {
+        if (!virtTestOOMActive() &&
+            (flags & FLAG_EXPECT_FAILURE)) {
            ret = 0;
            if (virTestGetDebug() > 1)
                fprintf(stderr, "Got expected error: %s\n",
@@ -371,7 +373,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
        goto out;
    }

-    if (!!virGetLastError() != !!(flags & FLAG_EXPECT_ERROR)) {
+    if (!virtTestOOMActive() &&
+        (!!virGetLastError() != !!(flags & FLAG_EXPECT_ERROR))) {
        if (virTestGetDebug() && (log = virtTestLogContentAndReset()))
            fprintf(stderr, "\n%s", log);
        goto out;
@@ -392,7 +395,8 @@ static int testCompareXMLToArgvFiles(const char *xml,
    }

 ok:
-    if (flags & FLAG_EXPECT_ERROR) {
+    if (!virtTestOOMActive() &&
+        (flags & FLAG_EXPECT_ERROR)) {
        /* need to suppress the errors */
        virResetLastError();
    }

--- a/tests/qemuxmlnstest.c
+++ b/tests/qemuxmlnstest.c
@@ -122,6 +122,7 @@ static int testCompareXMLToArgvFiles(const char *xml,
                                     &testCallbacks)))
        goto fail;

+    if (!virtTestOOMActive()) {
        if (!!virGetLastError() != expectError) {
            if (virTestGetDebug() && (log = virtTestLogContentAndReset()))
                fprintf(stderr, "\n%s", log);
@@ -132,6 +133,7 @@ static int testCompareXMLToArgvFiles(const char *xml,
            /* need to suppress the errors */
            virResetLastError();
        }
+    }

    if (!(actualargv = virCommandToString(cmd)))
        goto fail;

--- a/tests/testutils.c
+++ b/tests/testutils.c
@@ -47,6 +47,13 @@
 #include "virprocess.h"
 #include "virstring.h"

+#ifdef TEST_OOM
+# ifdef TEST_OOM_TRACE
+#  include <dlfcn.h>
+#  include <execinfo.h>
+# endif
+#endif
+
 #ifdef HAVE_PATHS_H
 # include <paths.h>
 #endif
@@ -64,12 +71,37 @@ static unsigned int testDebug = -1;
 static unsigned int testVerbose = -1;
 static unsigned int testExpensive = -1;

+#ifdef TEST_OOM
+static unsigned int testOOM = 0;
+static unsigned int testOOMStart = -1;
+static unsigned int testOOMEnd = -1;
+static unsigned int testOOMTrace = 0;
+# ifdef TEST_OOM_TRACE
+void *testAllocStack[30];
+int ntestAllocStack;
+# endif
+#endif
+static bool testOOMActive = false;
+
 static size_t testCounter = 0;
 static size_t testStart = 0;
 static size_t testEnd = 0;

 char *progname;

+bool virtTestOOMActive(void)
+{
+    return testOOMActive;
+}
+
+#ifdef TEST_OOM_TRACE
+static void virTestAllocHook(int nalloc ATTRIBUTE_UNUSED,
+                             void *opaque ATTRIBUTE_UNUSED)
+{
+    ntestAllocStack = backtrace(testAllocStack, ARRAY_CARDINALITY(testAllocStack));
+}
+#endif
+
 void virtTestResult(const char *name, int ret, const char *msg, ...)
 {
    va_list vargs;
@@ -108,6 +140,35 @@ void virtTestResult(const char *name, int ret, const char *msg, ...)
    va_end(vargs);
 }

+#ifdef TEST_OOM_TRACE
+static void
+virTestShowTrace(void)
+{
+    size_t j;
+    for (j = 2; j < ntestAllocStack; j++) {
+        Dl_info info;
+        char *cmd;
+
+        dladdr(testAllocStack[j], &info);
+        if (info.dli_fname &&
+            strstr(info.dli_fname, ".so")) {
+            if (virAsprintf(&cmd, ADDR2LINE " -f -e %s %p",
+                            info.dli_fname,
+                            ((void*)((unsigned long long)testAllocStack[j]
+                                     - (unsigned long long)info.dli_fbase))) < 0)
+                continue;
+        } else {
+            if (virAsprintf(&cmd, ADDR2LINE " -f -e %s %p",
+                            (char*)(info.dli_fname ? info.dli_fname : "<unknown>"),
+                            testAllocStack[j]) < 0)
+                continue;
+        }
+        ignore_value(system(cmd));
+        VIR_FREE(cmd);
+    }
+}
+#endif
+
 /*
 * Runs test
 *
@@ -163,6 +224,77 @@ virtTestRun(const char *title,
            fprintf(stderr, "!");
    }

+#ifdef TEST_OOM
+    if (testOOM && ret != EXIT_AM_SKIP) {
+        int nalloc;
+        int oomret;
+        int start, end;
+        size_t i;
+        virResetLastError();
+        virAllocTestInit();
+# ifdef TEST_OOM_TRACE
+        virAllocTestHook(virTestAllocHook, NULL);
+# endif
+        oomret = body(data);
+        nalloc = virAllocTestCount();
+        fprintf(stderr, "    Test OOM for nalloc=%d ", nalloc);
+        if (testOOMStart == -1 ||
+            testOOMEnd == -1) {
+            start = 0;
+            end = nalloc;
+        } else {
+            start = testOOMStart;
+            end = testOOMEnd + 1;
+        }
+        testOOMActive = true;
+        for (i = start; i < end; i++) {
+            bool missingFail = false;
+# ifdef TEST_OOM_TRACE
+            memset(testAllocStack, 0, ARRAY_CARDINALITY(testAllocStack));
+            ntestAllocStack = 0;
+# endif
+            virAllocTestOOM(i + 1, 1);
+            oomret = body(data);
+
+            /* fprintf() disabled because XML parsing APIs don't allow
+             * distinguish between element / attribute not present
+             * in the XML (which is non-fatal), vs OOM / malformed
+             * which should be fatal. Thus error reporting for
+             * optionally present XML is mostly broken.
+             */
+            if (oomret == 0) {
+                missingFail = true;
+# if 0
+                fprintf(stderr, " alloc %zu failed but no err status\n", i + 1);
+# endif
+            } else {
+                virErrorPtr lerr = virGetLastError();
+                if (!lerr) {
+# if 0
+                    fprintf(stderr, " alloc %zu failed but no error report\n", i + 1);
+# endif
+                    missingFail = true;
+                }
+            }
+            if ((missingFail && testOOMTrace) || (testOOMTrace > 1)) {
+                fprintf(stderr, "%s", "!");
+# ifdef TEST_OOM_TRACE
+                virTestShowTrace();
+# endif
+                ret = -1;
+            } else {
+                fprintf(stderr, "%s", ".");
+            }
+        }
+        testOOMActive = false;
+        if (ret == 0)
+            fprintf(stderr, " OK\n");
+        else
+            fprintf(stderr, " FAILED\n");
+        virAllocTestInit();
+    }
+#endif /* TEST_OOM */
+
    return ret;
 }

@@ -191,7 +323,6 @@ virtTestLoadFile(const char *file, char **buf)
    tmplen = buflen = st.st_size + 1;

    if (VIR_ALLOC_N(*buf, buflen) < 0) {
-        fprintf(stderr, "%s: larger than available memory (> %d)\n", file, buflen);
        VIR_FORCE_FCLOSE(fp);
        return -1;
    }
@@ -467,6 +598,7 @@ virtTestLogOutput(virLogSource source ATTRIBUTE_UNUSED,
 {
    struct virtTestLogData *log = data;
    virCheckFlags(VIR_LOG_STACK_TRACE,);
+    if (!testOOMActive)
        virBufferAsprintf(&log->buf, "%s: %s", timestamp, str);
 }

@@ -535,6 +667,9 @@ int virtTestMain(int argc,
 {
    int ret;
    char *testRange = NULL;
+#ifdef TEST_OOM
+    char *oomstr;
+#endif

    if (!virFileExists(abs_srcdir))
        return EXIT_AM_HARDFAIL;
@@ -590,6 +725,60 @@ int virtTestMain(int argc,
        }
    }

+#ifdef TEST_OOM
+    if ((oomstr = getenv("VIR_TEST_OOM")) != NULL) {
+        char *next;
+        if (testDebug == -1)
+            testDebug = 1;
+        testOOM = 1;
+        if (oomstr[0] != '\0' &&
+            oomstr[1] == ':') {
+            if (virStrToLong_ui(oomstr + 2, &next, 10, &testOOMStart) < 0) {
+                fprintf(stderr, "Cannot parse range %s\n", oomstr);
+                return EXIT_FAILURE;
+            }
+            if (*next == '\0') {
+                testOOMEnd = testOOMStart;
+            } else {
+                if (*next != '-') {
+                    fprintf(stderr, "Cannot parse range %s\n", oomstr);
+                    return EXIT_FAILURE;
+                }
+                if (virStrToLong_ui(next+1, NULL, 10, &testOOMEnd) < 0) {
+                    fprintf(stderr, "Cannot parse range %s\n", oomstr);
+                    return EXIT_FAILURE;
+                }
+            }
+        } else {
+            testOOMStart = -1;
+            testOOMEnd = -1;
+        }
+    }
+
+# ifdef TEST_OOM_TRACE
+    if ((oomstr = getenv("VIR_TEST_OOM_TRACE")) != NULL) {
+        if (virStrToLong_ui(oomstr, NULL, 10, &testOOMTrace) < 0) {
+            fprintf(stderr, "Cannot parse oom trace %s\n", oomstr);
+            return EXIT_FAILURE;
+        }
+    }
+# else
+    if (getenv("VIR_TEST_OOM_TRACE")) {
+        fprintf(stderr, "%s", "OOM test tracing not enabled in this build\n");
+        return EXIT_FAILURE;
+    }
+# endif
+#else /* TEST_OOM */
+    if (getenv("VIR_TEST_OOM")) {
+        fprintf(stderr, "%s", "OOM testing not enabled in this build\n");
+        return EXIT_FAILURE;
+    }
+    if (getenv("VIR_TEST_OOM_TRACE")) {
+        fprintf(stderr, "%s", "OOM test tracing not enabled in this build\n");
+        return EXIT_FAILURE;
+    }
+#endif /* TEST_OOM */
+
    ret = (func)();

    virResetLastError();

--- a/tests/testutils.h
+++ b/tests/testutils.h
@@ -44,6 +44,8 @@ extern char *progname;
 #  error Fix Makefile.am
 # endif

+bool virtTestOOMActive(void);
+
 void virtTestResult(const char *name, int ret, const char *msg, ...)
    ATTRIBUTE_FMT_PRINTF(3,4);
 int virtTestRun(const char *title,