[cmake] Run fuzzing and shaping tests (#666)

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
       - image: fedora
     steps:
       - checkout
-      - run: dnf install -y gcc ragel cmake make which glib2-devel freetype-devel cairo-devel libicu-devel graphite2-devel wget tar bzip2 || true
+      - run: dnf install -y gcc ragel cmake make which glib2-devel freetype-devel cairo-devel libicu-devel graphite2-devel wget tar bzip2 python || true
       - run: wget http://$ODSUSER:$ODSPASS@behdad.org/harfbuzz-private/OracleDeveloperStudio12.6-linux-x86-bin.tar.bz2 && tar xf OracleDeveloperStudio12.6-linux-x86-bin.tar.bz2 --owner root --group root --no-same-owner
       - run: CC=/root/project/OracleDeveloperStudio12.6-linux-x86-bin/developerstudio12.6/bin/suncc CXX=/root/project/OracleDeveloperStudio12.6-linux-x86-bin/developerstudio12.6/bin/sunCC cmake -DHB_HAVE_GRAPHITE2=ON -DHB_BUILTIN_UCDN=ON -DHB_HAVE_GLIB=ON -DHB_HAVE_ICU=ON -DHB_HAVE_FREETYPE=ON -Bbuild -H.
       - run: make -Cbuild

CMakeLists.txt

@@ -92,45 +92,56 @@ endif ()
 
 
 ## Extract variables from Makefile files
-# http://stackoverflow.com/a/27630120/1414809
-function (prepend var prefix)
+function (extract_make_variable variable makefile_source)
+  string(REGEX MATCH "${variable} = ([^$]+)\\$" temp ${makefile_source})
+  string(REGEX MATCHALL "[^ \n\t\\]+" listVar ${CMAKE_MATCH_1})
+  set (${variable} ${listVar} PARENT_SCOPE)
+endfunction ()
+
+# http://stackoverflow.com/a/27630120
+function (add_prefix_to_list var prefix)
   set (listVar "")
-  foreach (f ${ARGN})
+  foreach (f ${${var}})
     list(APPEND listVar "${prefix}${f}")
   endforeach ()
   set (${var} "${listVar}" PARENT_SCOPE)
 endfunction ()
 
-function (extract_make_variable variable file prefix)
-  string(REGEX MATCH "${variable} = ([^$]+)\\$" temp ${file})
-  string(REGEX MATCHALL "[^ \n\t\\]+" list ${CMAKE_MATCH_1})
-  prepend(list ${prefix} ${list})
-  set (${variable} ${list} PARENT_SCOPE)
-endfunction ()
-
 file(READ ${PROJECT_SOURCE_DIR}/src/Makefile.sources SRCSOURCES)
 file(READ ${PROJECT_SOURCE_DIR}/util/Makefile.sources UTILSOURCES)
 file(READ ${PROJECT_SOURCE_DIR}/src/hb-ucdn/Makefile.sources UCDNSOURCES)
 
-extract_make_variable(HB_BASE_sources ${SRCSOURCES} "${PROJECT_SOURCE_DIR}/src/")
-extract_make_variable(HB_BASE_headers ${SRCSOURCES} "${PROJECT_SOURCE_DIR}/src/")
-extract_make_variable(HB_FALLBACK_sources ${SRCSOURCES} "${PROJECT_SOURCE_DIR}/src/")
-extract_make_variable(HB_OT_sources ${SRCSOURCES} "${PROJECT_SOURCE_DIR}/src/")
-extract_make_variable(HB_OT_headers ${SRCSOURCES} "${PROJECT_SOURCE_DIR}/src/")
-
+extract_make_variable(HB_BASE_sources ${SRCSOURCES})
+add_prefix_to_list(HB_BASE_sources "${PROJECT_SOURCE_DIR}/src/")
+extract_make_variable(HB_BASE_headers ${SRCSOURCES})
+add_prefix_to_list(HB_BASE_headers "${PROJECT_SOURCE_DIR}/src/")
+extract_make_variable(HB_FALLBACK_sources ${SRCSOURCES})
+add_prefix_to_list(HB_FALLBACK_sources "${PROJECT_SOURCE_DIR}/src/")
+extract_make_variable(HB_OT_sources ${SRCSOURCES})
+add_prefix_to_list(HB_OT_sources "${PROJECT_SOURCE_DIR}/src/")
+extract_make_variable(HB_OT_headers ${SRCSOURCES})
+add_prefix_to_list(HB_OT_headers "${PROJECT_SOURCE_DIR}/src/")
+
+extract_make_variable(HB_BASE_RAGEL_GENERATED_sources ${SRCSOURCES})
+extract_make_variable(HB_OT_RAGEL_GENERATED_sources ${SRCSOURCES})
 if (IN_HB_DIST)
-  set (RAGEL_GENERATED_DIR "${PROJECT_SOURCE_DIR}/src/")
+  add_prefix_to_list(HB_BASE_RAGEL_GENERATED_sources "${PROJECT_SOURCE_DIR}/src/")
+  add_prefix_to_list(HB_OT_RAGEL_GENERATED_sources "${PROJECT_SOURCE_DIR}/src/")
 else ()
-  set (RAGEL_GENERATED_DIR "${PROJECT_BINARY_DIR}/src/")
+  add_prefix_to_list(HB_BASE_RAGEL_GENERATED_sources "${PROJECT_BINARY_DIR}/src/")
+  add_prefix_to_list(HB_OT_RAGEL_GENERATED_sources "${PROJECT_BINARY_DIR}/src/")
 endif ()
-extract_make_variable(HB_BASE_RAGEL_GENERATED_sources ${SRCSOURCES} ${RAGEL_GENERATED_DIR})
-extract_make_variable(HB_OT_RAGEL_GENERATED_sources ${SRCSOURCES} ${RAGEL_GENERATED_DIR})
 
-extract_make_variable(HB_VIEW_sources ${UTILSOURCES} "${PROJECT_SOURCE_DIR}/util/")
-extract_make_variable(HB_SHAPE_sources ${UTILSOURCES} "${PROJECT_SOURCE_DIR}/util/")
-extract_make_variable(HB_OT_SHAPE_CLOSURE_sources ${UTILSOURCES} "${PROJECT_SOURCE_DIR}/util/")
+extract_make_variable(HB_VIEW_sources ${UTILSOURCES})
+add_prefix_to_list(HB_VIEW_sources "${PROJECT_SOURCE_DIR}/util/")
+extract_make_variable(HB_SHAPE_sources ${UTILSOURCES})
+add_prefix_to_list(HB_SHAPE_sources "${PROJECT_SOURCE_DIR}/util/")
+extract_make_variable(HB_OT_SHAPE_CLOSURE_sources ${UTILSOURCES})
+add_prefix_to_list(HB_OT_SHAPE_CLOSURE_sources "${PROJECT_SOURCE_DIR}/util/")
+
+extract_make_variable(LIBHB_UCDN_sources ${UCDNSOURCES})
+add_prefix_to_list(LIBHB_UCDN_sources "${PROJECT_SOURCE_DIR}/src/hb-ucdn/")
 
-extract_make_variable(LIBHB_UCDN_sources ${UCDNSOURCES} "${PROJECT_SOURCE_DIR}/src/hb-ucdn/")
 
 file(READ configure.ac CONFIGUREAC)
 string(REGEX MATCH "\\[(([0-9]+)\\.([0-9]+)\\.([0-9]+))\\]" HB_VERSION_MATCH ${CONFIGUREAC})

appveyor.yml

@@ -41,9 +41,9 @@ build_script:
   - 'if "%compiler%"=="msvc" if not "%platform%"=="ARM" cmake -DHB_HAVE_UNISCRIBE=ON -DHB_HAVE_DIRECTWRITE=ON -DHB_HAVE_GLIB=ON -DHB_HAVE_FREETYPE=ON -G "%generator%" -DCMAKE_TOOLCHAIN_FILE=c:/tools/vcpkg/scripts/buildsystems/vcpkg.cmake ../'
 
   - 'if "%compiler%"=="msvc" msbuild harfbuzz.sln /p:Configuration=%configuration% /p:Platform=%platform%'
-  - 'if "%compiler%"=="msvc" if not "%platform%"=="ARM" ctest'
+  - 'if "%compiler%"=="msvc" if not "%platform%"=="ARM" ctest --output-on-failure'
 
-  - 'if "%compiler%"=="msys2" C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S mingw-w64-$MSYS2_ARCH-{freetype,cairo,icu,gettext,gobject-introspection,gcc,gcc-libs,glib2,graphite2,pkg-config}"'
+  - 'if "%compiler%"=="msys2" C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S mingw-w64-$MSYS2_ARCH-{freetype,cairo,icu,gettext,gobject-introspection,gcc,gcc-libs,glib2,graphite2,pkg-config,python2}"'
   - 'if "%compiler%"=="msys2" C:\msys64\usr\bin\bash -lc "cd $APPVEYOR_BUILD_FOLDER; PATH=$PATH:/mingw64/bin:/mingw32/bin; ./autogen.sh --with-uniscribe --with-freetype --with-glib --with-gobject --with-cairo --with-icu --with-graphite2 --build=%MINGW_CHOST% --host=%MINGW_CHOST% --prefix=%MINGW_PREFIX%; make; make check || (cat */test-suite.log test/*/test-suite.log && false)"'
 
 cache:

test/CMakeLists.txt

 add_subdirectory(api)
+add_subdirectory(shaping)
+# Not ready yet
+if (NOT MSVC)
+  add_subdirectory(fuzzing)
+endif ()

test/api/CMakeLists.txt

-macro (_add_tests)
-  foreach(test_name ${ARGV})
+if (HB_HAVE_GLIB)
+  file (READ "${CMAKE_CURRENT_SOURCE_DIR}/Makefile.am" MAKEFILEAM)
+  extract_make_variable (TEST_PROGS ${MAKEFILEAM})
+
+  list (APPEND TEST_PROGS
+    test-ot-tag
+    test-c
+    test-cplusplus
+  )
+
+  if (HB_HAVE_FREETYPE)
+    list (APPEND TEST_PROGS test-ot-math)
+  endif ()
+
+  foreach (test_name IN ITEMS ${TEST_PROGS})
     if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${test_name}.c)
       add_executable (${test_name} ${test_name}.c)
     elseif (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${test_name}.cc)
       add_executable (${test_name} ${test_name}.cc)
     else ()
-      message(FATAL_ERROR "No source file found for test ${test_name}")
+      message (FATAL_ERROR "No source file found for test ${test_name}")
     endif ()
     target_link_libraries (${test_name} harfbuzz)
     add_test (${test_name} ${test_name})
   endforeach ()
-  set_tests_properties (${ARGV} PROPERTIES ENVIRONMENT
+  set_tests_properties (${TEST_PROGS} PROPERTIES ENVIRONMENT
     "G_TEST_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};G_TEST_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}"
-    )
-endmacro ()
-
-if (HB_HAVE_GLIB)
-  _add_tests (
-    test-blob
-    test-buffer
-    test-common
-    test-font
-    test-object
-    test-set
-    test-shape
-    test-unicode
-    test-version
-    test-ot-tag
-    test-c
-    test-cplusplus
-    )
-
-  if (HB_HAVE_FREETYPE)
-    _add_tests (
-      test-ot-math
-      )
-  endif (HB_HAVE_FREETYPE)
-endif (HB_HAVE_GLIB)
+  )
+endif ()

test/fuzzing/CMakeLists.txt

+if (HB_HAVE_GLIB)
+  add_executable (hb-fuzzer hb-fuzzer.cc hb-fuzzer.hh main.cc)
+  target_link_libraries (hb-fuzzer harfbuzz)
+  add_test (hb-fuzzer python run-fuzzer-tests.py)
+  set_tests_properties (hb-fuzzer PROPERTIES
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    ENVIRONMENT "top_builddir=${PROJECT_BINARY_DIR};testdir=test;fuzzingdir=fuzzing"
+  )
+endif ()

test/fuzzing/Makefile.am

@@ -14,6 +14,8 @@ $(top_builddir)/src/libharfbuzz-fuzzing.la: lib
 
 EXTRA_DIST += \
 	README \
+	run-fuzzer-tests.py \
+	CMakeLists.txt \
 	$(NULL)
 
 check_PROGRAMS = \
@@ -44,6 +46,22 @@ hb_fuzzer_DEPENDENCIES = \
 	lib \
 	$(NULL)
 
+AM_TESTS_ENVIRONMENT = \
+	EXEEXT="$(EXEEXT)"; \
+	export EXEEXT; \
+	srcdir="$(srcdir)"; \
+	export srcdir; \
+	builddir="$(builddir)"; \
+	export builddir; \
+	$(NULL)
+
+if AUTOMAKE_OLDER_THAN_1_13
+TESTS_ENVIRONMENT = \
+	$(AM_TESTS_ENVIRONMENT) \
+	$(TESTS_LOG_COMPILER) \
+	$(NULL)
+endif
+
 check:
 	cat $(srcdir)/../shaping/tests/fuzzed.tests | \
 	cut -d: -f1 | while read x; do \

test/fuzzing/run-fuzzer-tests.py

+#!/usr/bin/env python
+
+from __future__ import print_function
+import sys, os, subprocess
+
+srcdir = os.environ.get ("srcdir", ".")
+EXEEXT = os.environ.get ("EXEEXT", "")
+top_builddir = os.environ.get ("top_builddir", ".")
+testdir = os.environ.get ("testdir", ".")
+fuzzingdir = os.environ.get ("fuzzingdir", ".")
+hb_fuzzer = os.path.join (top_builddir, testdir, fuzzingdir, "hb-fuzzer" + EXEEXT)
+fails = 0
+
+for line in open (os.path.join (srcdir, "..", "shaping", "tests", "fuzzed.tests")):
+	font = line.split (":")[0]
+
+	p = subprocess.Popen (
+		[hb_fuzzer, os.path.join (srcdir, "..", "shaping", font)],
+		stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+	if p.wait () != 0:
+		fails = fails + 1
+
+	print ((p.stdout.read () + p.stderr.read ()).decode ("utf-8").strip ())
+
+if fails:
+	print ("%i fuzzer related tests failed." % fails)
+	sys.exit (1)

test/shaping/CMakeLists.txt

+if (HB_BUILD_UTILS)
+  file (READ "${CMAKE_CURRENT_SOURCE_DIR}/Makefile.am" MAKEFILEAM)
+  extract_make_variable (TESTS ${MAKEFILEAM})
+  foreach (test IN ITEMS ${TESTS})
+    add_test (${test} python run-tests.py "${test}")
+    set_tests_properties (${test} PROPERTIES
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      ENVIRONMENT "top_builddir=${PROJECT_BINARY_DIR};utildir=."
+    )
+  endforeach ()
+endif ()

test/shaping/Makefile.am

@@ -29,6 +29,7 @@ EXTRA_DIST += \
 	run-tests.py \
 	texts/in-tree \
 	fonts/sha1sum \
+	CMakeLists.txt \
 	$(TESTS) \
 	$(NULL)
 
@@ -104,7 +105,7 @@ TESTS_ENVIRONMENT = \
 	$(NULL)
 endif
 
-TESTS_LOG_COMPILER = python $(srcdir)/run-tests.py
+TESTS_LOG_COMPILER = $(srcdir)/run-tests.py
 
 .PHONY: manifests
 

test/shaping/run-tests.py

@@ -22,10 +22,11 @@ srcdir = os.environ.get ("srcdir", ".")
 builddir = os.environ.get ("builddir", ".")
 top_builddir = os.environ.get ("top_builddir",
 	os.path.normpath (os.path.join (os.getcwd (), "..", "..")))
+utildir = os.environ.get ("utildir", "util")
 EXEEXT = os.environ.get ("EXEEXT", "")
 
 extra_options = "--verify"
-hb_shape = os.path.join (top_builddir, "util", "hb-shape" + EXEEXT)
+hb_shape = os.path.join (top_builddir, utildir, "hb-shape" + EXEEXT)
 
 fails = 0
 args = sys.argv[1:]