Win32/NMake builds: Support builds from GIT (#498)

Add Python scripts to generate the full win32/config.h.win32 and
src/hb-version.h which can be used to build directly from a GIT
checkout.  Since the scripts are currently intended for building from a
GIT checkout, these are not distributed in the release tarballs.

Also, support the re-build of Ragel-generated .hh headers using the NMake
build system, and allow one to specify the path of the Ragel executable
if a suitable one cannot be found in the PATH.

Update the Win32/NMake build documentation to let people know about how
these mechanisms can be utilized.

win32/README.txt

@@ -11,7 +11,22 @@ backends are enabled, and this is the base configuration that is built if no
 options (see below) are specified.  A 'clean' target is provided-it is recommended
 that one cleans the build and redo the build if any configuration option changed.
 An 'install' target is also provided to copy the built items in their appropriate
-locations under $(PREFIX), which is described below.
+locations under $(PREFIX), which is described below.  A 'reallyclean' target is
+provided that not only does what is done for the 'clean' target, but also removes
+the sources/headers that are generated from the Ragel sources.  Therefore, if one
+is not building from a release tarball, or is rebuilding after using the 'reallyclean'
+target or when the Ragel (*.rl) sources are updated, the Ragel state machine
+compiler (ragel.exe) is needed, and needs to be passed in via RAGEL=<path_to_ragel_exe>
+if ragel.exe is not already in the PATH.
+
+We now support building from a GIT checkout via NMake for convenience.  In addition to
+the requirements as outlined in the later part of this file, you will need to run the
+setup.py (Python 2.7.x or later) script to generate the headers (src\hb-version.h and
+win32\config.h.win32) that are normally shipped in a release tarball before running
+NMake, and you will need to pass RAGEL=<path_to_ragel_exe> if the Ragel state machine
+compiler (ragel.exe) is not in your PATH when invoking NMake.  Note that the
+'reallyclean' target does not remove these 2 generated headers, so re-run the setup.py
+script if necessary.
 
 Invoke the build by issuing the command:
 nmake /f Makefile.vc CFG=[release|debug] [PREFIX=...] <option1=1 option2=1 ...>
@@ -32,6 +47,7 @@ PREFIX: Optional.  Base directory of where the third-party headers, libraries
         2012: 11
         2013: 12
         2015: 14
+        2017: 15
 
 Explanation of options, set by <option>=1:
 ------------------------------------------
@@ -75,4 +91,6 @@ PYTHON: Full path to the Python interpretor to be used, if it is not in %PATH%.
 
 PERL: Full path to the PERL interpretor to be used, if it is not in %PATH%.
 
+RAGEL: Full path to the Ragel state machine compiler executable, if not in %PATH%
+
 LIBTOOL_DLL_NAME: Enable libtool-style DLL names.

win32/build-rules-msvc.mak

@@ -52,7 +52,7 @@ $(CFG)\$(PLAT)\harfbuzz-gobject.lib: $(HARFBUZZ_GOBJECT_DLL_FILENAME).dll
 # $(dependent_objects)
 # <<
 # 	@-if exist $@.manifest mt /manifest $@.manifest /outputresource:$@;2
-$(HARFBUZZ_DLL_FILENAME).dll: config.h $(harfbuzz_dll_OBJS) $(CFG)\$(PLAT)\harfbuzz
+$(HARFBUZZ_DLL_FILENAME).dll: config.h $(HB_RAGEL_GENERATED_ACTUAL_SOURCES) $(harfbuzz_dll_OBJS) $(CFG)\$(PLAT)\harfbuzz
 	link /DLL $(LDFLAGS) $(HB_DEP_LIBS) /implib:$(CFG)\$(PLAT)\harfbuzz.lib -out:$@ @<<
 $(harfbuzz_dll_OBJS)
 <<
@@ -125,3 +125,6 @@ clean:
 	@-if exist $(CFG)\$(PLAT)\harfbuzz-gobject\hb-gobject-enums.cc del $(CFG)\$(PLAT)\harfbuzz-gobject\hb-gobject-enums.cc
 	@-del vc$(VSVER)0.pdb
 	@-del config.h
+
+reallyclean: clean
+	@-del /f /q $(HB_RAGEL_GENERATED_ACTUAL_SOURCES)

win32/config-msvc.mak

@@ -30,6 +30,11 @@ UNISCRIBE_LIB = usp10.lib gdi32.lib rpcrt4.lib user32.lib
 # Directwrite is needed for DirectWrite shaping support
 DIRECTWRITE_LIB = dwrite.lib
 
+# Full path to Ragel state machine compiler if not already in PATH
+!if "$(RAGEL)" == ""
+RAGEL = ragel
+!endif
+
 # Please do not change anything beneath this line unless maintaining the NMake Makefiles
 # Bare minimum features and sources built into HarfBuzz on Windows
 HB_DEFINES =
@@ -48,6 +53,8 @@ HB_HEADERS =	\
 	$(HB_NODIST_headers)	\
 	$(HB_OT_headers)
 
+RAGEL_RAW_GEN_SRCS = $(HB_OT_RAGEL_GENERATED_sources) $(HB_BASE_RAGEL_GENERATED_sources)
+
 # Minimal set of (system) libraries needed for the HarfBuzz DLL
 HB_DEP_LIBS =
 

win32/create-lists-msvc.mak

@@ -132,6 +132,15 @@ NULL=
 !if [call create-lists.bat footer hb_srcs.mak]
 !endif
 
+!if [call create-lists.bat header hb_srcs.mak HB_RAGEL_GENERATED_ACTUAL_SOURCES]
+!endif
+
+!if [for %s in ($(RAGEL_RAW_GEN_SRCS)) do @call create-lists.bat file hb_srcs.mak ..\src\%s]
+!endif
+
+!if [call create-lists.bat footer hb_srcs.mak]
+!endif
+
 !include hb_srcs.mak
 
 !if [del /f /q hb_srcs.mak]

win32/generate-msvc.mak

@@ -24,3 +24,9 @@ $(HB_GOBJECT_ENUM_GENERATED_SOURCES): ..\src\hb-gobject-enums.h.tmpl ..\src\hb-g
 # Create the build directories
 $(CFG)\$(PLAT)\harfbuzz $(CFG)\$(PLAT)\harfbuzz-gobject $(CFG)\$(PLAT)\util:
 	@-md $@
+
+.SUFFIXES: .c .cc .hh .rl
+
+# Generate headers from Ragel sources
+{..\src\}.rl{..\src\}.hh:
+	$(RAGEL) -e -F1 -o $@ $<

win32/info-msvc.mak

@@ -133,10 +133,16 @@ help:
 	@echo NO_UCDN:
 	@echo Do not use the bundled Unicode callback, which is the default.  GLib or
 	@echo ICU-based unicode callback is therefore required.
-	@echo
+	@echo.
 	@echo UNISCRIBE:
 	@echo Enable Uniscribe support.
 	@echo.
+	@echo RAGEL:
+	@echo Set the full path to the Ragel state machine compiler, if not already in
+	@echo PATH.  The Ragel state machine compiler is required if not building from
+	@echo a release tarball, or a rebuild is to be carried out after using the
+	@echo 'reallyclean' target.
+	@echo.
 	@echo Note that GLib2 support is required for all utility and test programs.
 	@echo ======
 	@echo A 'clean' target is supported to remove all generated files, intermediate

win32/pc_base.py

+#!/usr/bin/python
+#
+# Simple utility script to generate the basic info
+# needed in a .pc (pkg-config) file, used especially
+# for introspection purposes
+
+# This can be used in various projects where
+# there is the need to generate .pc files,
+# and is copied from GLib's $(srcroot)/win32
+
+# Author: Fan, Chun-wei
+# Date: March 10, 2016
+
+import os
+import sys
+import argparse
+
+class BasePCItems:
+    def __init__(self):
+        self.base_replace_items = {}
+        self.exec_prefix = ''
+        self.includedir = ''
+        self.libdir = ''
+        self.prefix = ''
+        self.srcdir = os.path.dirname(__file__)
+        self.top_srcdir = self.srcdir + '\\..'
+        self.version = ''
+
+    def setup(self, argv, parser=None):
+        if parser is None:
+            parser = argparse.ArgumentParser(description='Setup basic .pc file info')
+        parser.add_argument('--prefix', help='prefix of the installed library',
+                            required=True)
+        parser.add_argument('--exec-prefix',
+                            help='prefix of the installed programs, \
+                                  if different from the prefix')
+        parser.add_argument('--includedir',
+                            help='includedir of the installed library, \
+                                  if different from ${prefix}/include')
+        parser.add_argument('--libdir',
+                            help='libdir of the installed library, \
+                                  if different from ${prefix}/lib')
+        parser.add_argument('--version', help='Version of the package',
+                            required=True)
+        args = parser.parse_args()
+
+        self.version = args.version
+
+        # check whether the prefix and exec_prefix are valid
+        if not os.path.exists(args.prefix):
+            raise SystemExit('Specified prefix \'%s\' is invalid' % args.prefix)
+
+        # use absolute paths for prefix
+        self.prefix = os.path.abspath(args.prefix).replace('\\','/')
+
+        # check and setup the exec_prefix
+        if getattr(args, 'exec_prefix', None) is None:
+            exec_prefix_use_shorthand = True
+            self.exec_prefix = '${prefix}'
+        else:
+            if args.exec_prefix.startswith('${prefix}'):
+                exec_prefix_use_shorthand = True
+                input_exec_prefix = args.prefix + args.exec_prefix[len('${prefix}'):]
+            else:
+                exec_prefix_use_shorthand = False
+                input_exec_prefix = args.exec_prefix
+            if not os.path.exists(input_exec_prefix):
+                raise SystemExit('Specified exec_prefix \'%s\' is invalid' %
+                                  args.exec_prefix)
+            if exec_prefix_use_shorthand is True:
+                self.exec_prefix = args.exec_prefix.replace('\\','/')
+            else:
+                self.exec_prefix = os.path.abspath(input_exec_prefix).replace('\\','/')
+
+        # check and setup the includedir
+        if getattr(args, 'includedir', None) is None:
+            self.includedir = '${prefix}/include'
+        else:
+            if args.includedir.startswith('${prefix}'):
+                includedir_use_shorthand = True
+                input_includedir = args.prefix + args.includedir[len('${prefix}'):]
+            else:
+                if args.includedir.startswith('${exec_prefix}'):
+                    includedir_use_shorthand = True
+                    input_includedir = input_exec_prefix + args.includedir[len('${exec_prefix}'):]
+                else:
+                    includedir_use_shorthand = False
+                    input_includedir = args.includedir
+            if not os.path.exists(input_includedir):
+                raise SystemExit('Specified includedir \'%s\' is invalid' %
+                                  args.includedir)
+            if includedir_use_shorthand is True:
+                self.includedir = args.includedir.replace('\\','/')
+            else:
+                self.includedir = os.path.abspath(input_includedir).replace('\\','/')
+
+        # check and setup the libdir
+        if getattr(args, 'libdir', None) is None:
+            self.libdir = '${prefix}/lib'
+        else:
+            if args.libdir.startswith('${prefix}'):
+                libdir_use_shorthand = True
+                input_libdir = args.prefix + args.libdir[len('${prefix}'):]
+            else:
+                if args.libdir.startswith('${exec_prefix}'):
+                    libdir_use_shorthand = True
+                    input_libdir = input_exec_prefix + args.libdir[len('${exec_prefix}'):]
+                else:
+                    libdir_use_shorthand = False
+                    input_libdir = args.libdir
+            if not os.path.exists(input_libdir):
+                raise SystemExit('Specified libdir \'%s\' is invalid' %
+			                      args.libdir)
+            if libdir_use_shorthand is True:
+                self.libdir = args.libdir.replace('\\','/')
+            else:
+                self.libdir = os.path.abspath(input_libdir).replace('\\','/')
+
+        # setup dictionary for replacing items in *.pc.in
+        self.base_replace_items.update({'@VERSION@': self.version})
+        self.base_replace_items.update({'@prefix@': self.prefix})
+        self.base_replace_items.update({'@exec_prefix@': self.exec_prefix})
+        self.base_replace_items.update({'@libdir@': self.libdir})
+        self.base_replace_items.update({'@includedir@': self.includedir})

win32/replace.py

+#!/usr/bin/python
+#
+# Simple utility script to manipulate
+# certain types of strings in a file
+
+# This can be used in various projects where
+# there is the need to replace strings in files,
+# and is copied from GLib's $(srcroot)/win32
+
+# Author: Fan, Chun-wei
+# Date: September 03, 2014
+
+import os
+import sys
+import re
+import string
+import argparse
+
+valid_actions = ['remove-prefix',
+                 'replace-var',
+                 'replace-str',
+                 'remove-str']
+
+def open_file(filename, mode):
+    if sys.version_info[0] < 3:
+        return open(filename, mode=mode)
+    else:
+        return open(filename, mode=mode, encoding='utf-8')
+
+def replace_multi(src, dest, replace_items):
+    with open_file(src, 'r') as s:
+        with open_file(dest, 'w') as d:
+            for line in s:
+                replace_dict = dict((re.escape(key), value) \
+                               for key, value in replace_items.items())
+                replace_pattern = re.compile("|".join(replace_dict.keys()))
+                d.write(replace_pattern.sub(lambda m: \
+                        replace_dict[re.escape(m.group(0))], line))
+
+def replace(src, dest, instring, outstring):
+    replace_item = {instring: outstring}
+    replace_multi(src, dest, replace_item)
+
+def check_required_args(args, params):
+    for param in params:
+        if getattr(args, param, None) is None:
+            raise SystemExit('%s: error: --%s argument is required' % (__file__, param))
+
+def warn_ignored_args(args, params):
+    for param in params:
+        if getattr(args, param, None) is not None:
+            print('%s: warning: --%s argument is ignored' % (__file__, param))
+
+def main(argv):
+
+    parser = argparse.ArgumentParser(description='Process strings in a file.')
+    parser.add_argument('-a',
+                        '--action',
+                        help='Action to carry out.  Can be one of:\n'
+                             'remove-prefix\n'
+                             'replace-var\n'
+                             'replace-str\n'
+                             'remove-str',
+                        choices=valid_actions)
+    parser.add_argument('-i', '--input', help='Input file')
+    parser.add_argument('-o', '--output', help='Output file')
+    parser.add_argument('--instring', help='String to replace or remove')
+    parser.add_argument('--var', help='Autotools variable name to replace')
+    parser.add_argument('--outstring',
+                        help='New String to replace specified string or variable')
+    parser.add_argument('--removeprefix', help='Prefix of string to remove')
+
+    args = parser.parse_args()
+
+    input_string = ''
+    output_string = ''
+
+    # We must have action, input, output for all operations
+    check_required_args(args, ['action','input','output'])
+
+    # Build the arguments by the operation that is to be done,
+    # to be fed into replace()
+
+    # Get rid of prefixes from a string
+    if args.action == 'remove-prefix':
+        check_required_args(args, ['instring','removeprefix'])
+        warn_ignored_args(args, ['outstring','var'])
+        input_string = args.removeprefix + args.instring
+        output_string = args.instring
+
+    # Replace an m4-style variable (those surrounded by @...@)
+    if args.action == 'replace-var':
+        check_required_args(args, ['var','outstring'])
+        warn_ignored_args(args, ['instring','removeprefix'])
+        input_string = '@' + args.var + '@'
+        output_string = args.outstring
+
+    # Replace a string
+    if args.action == 'replace-str':
+        check_required_args(args, ['instring','outstring'])
+        warn_ignored_args(args, ['var','removeprefix'])
+        input_string = args.instring
+        output_string = args.outstring
+
+    # Remove a string
+    if args.action == 'remove-str':
+        check_required_args(args, ['instring'])
+        warn_ignored_args(args, ['var','outstring','removeprefix'])
+        input_string = args.instring
+        output_string = ''
+
+    replace(args.input, args.output, input_string, output_string)
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

win32/setup.py

+#!/usr/bin/python
+# vim: encoding=utf-8
+#expand *.in files
+#this script is only intended for building from git, not for building from the released tarball, which already includes all necessary files
+import os
+import sys
+import re
+import string
+import subprocess
+import optparse
+from pc_base import BasePCItems
+from replace import replace_multi
+
+def get_version_items(srcroot):
+    ver = {}
+    RE_VERSION_LINE_START = re.compile(r'^AC_INIT\(\[(.+)\], *\n')
+    RE_VERSION_LINE_BODY = re.compile(r'^        \[(.+)\], *\n')
+    RE_VERSION_LINE_END = re.compile(r'^        \[(.+)\]\) *\n')
+
+    # Read from the AC_INIT lines to get the version/name/URLs info
+    with open(os.path.join(srcroot, 'configure.ac'), 'r') as ac:
+        for i in ac:
+            mo_init = RE_VERSION_LINE_START.search(i)
+            mo_pkg_info = RE_VERSION_LINE_BODY.search(i)
+            mo_pkg_url = RE_VERSION_LINE_END.search(i)
+            if mo_init:
+                ver['@PACKAGE_NAME@'] = mo_init.group(1) 
+            if mo_pkg_info:
+                if mo_pkg_info.group(1).startswith('http'):
+                    ver['@PACKAGE_BUGREPORT@'] = mo_pkg_info.group(1)
+                elif mo_pkg_info.group(1)[0].isdigit():
+                    ver['@PACKAGE_VERSION@'] = mo_pkg_info.group(1)
+                else:
+                    ver['@PACKAGE_TARNAME@'] = mo_pkg_info.group(1)
+            if mo_pkg_url:
+                ver['@PACKAGE_URL@'] = mo_pkg_url.group(1)
+            
+    ver['@HB_VERSION@'] = ver['@PACKAGE_VERSION@']
+
+    pkg_ver_parts = ver['@PACKAGE_VERSION@'].split('.')
+    ver['@HB_VERSION_MAJOR@'] = pkg_ver_parts[0]
+    ver['@HB_VERSION_MINOR@'] = pkg_ver_parts[1]
+    ver['@HB_VERSION_MICRO@'] = pkg_ver_parts[2]
+    return ver
+
+def main(argv):
+    pc = BasePCItems()
+    srcroot = pc.top_srcdir
+    srcdir = pc.srcdir
+    ver = get_version_items(srcroot)
+
+    replace_multi(os.path.join(srcdir, 'config.h.win32.in'),
+                  os.path.join(srcdir, 'config.h.win32'),
+                  ver)
+
+    replace_multi(os.path.join(srcroot, 'src', 'hb-version.h.in'),
+                  os.path.join(srcroot, 'src', 'hb-version.h'),
+                  ver)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))