Merge branch 'master' into cff-subset

.circleci/config.yml

@@ -126,7 +126,7 @@ jobs:
       - run: update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.lld" 10
       - run: wget https://ftp.gnome.org/pub/gnome/sources/glib/2.58/glib-2.58.1.tar.xz && tar xf glib-2.58.1.tar.xz && cd glib-2.58.1 && ./autogen.sh --with-pcre CPPFLAGS="-fsanitize=memory" LDFLAGS="-fsanitize=memory" CFLAGS="-fsanitize=memory" CXXFLAGS="-fsanitize=memory" LD=ld.lld CC=clang CXX=clang++ && make -j32 && make install && cd ..
       - run: wget http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2 && tar xf freetype-2.9.tar.bz2 && cd freetype-2.9 && ./autogen.sh && ./configure CPPFLAGS="-fsanitize=memory" LDFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" CFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" CXXFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" LD=ld.lld CC=clang CXX=clang++ && make -j32 && make install && cd ..
-      - run: CPPFLAGS="-fsanitize=memory" LDFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" CFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" CXXFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" LD=ld.lld CC=clang CXX=clang++ ./autogen.sh --with-freetype --with-glib --without-icu
+      - run: CPPFLAGS="-fsanitize=memory -fsanitize-memory-track-origins" LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins -O1 -g -fno-omit-frame-pointer" CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins -O1 -g -fno-omit-frame-pointer" CXXFLAGS="-fsanitize=memory -fsanitize-memory-track-origins -O1 -g -fno-omit-frame-pointer" LD=ld.lld CC=clang CXX=clang++ ./autogen.sh --with-freetype --with-glib --without-icu
       - run: make -j32 && MSAN_OPTIONS=exitcode=42 make check || .ci/fail.sh | asan_symbolize | c++filt
 
   clang-tsan:

docs/harfbuzz-sections.txt

@@ -67,6 +67,8 @@ hb_buffer_set_user_data
 hb_buffer_get_user_data
 hb_buffer_get_glyph_infos
 hb_buffer_get_glyph_positions
+hb_buffer_get_invisible_glyph
+hb_buffer_set_invisible_glyph
 hb_buffer_set_replacement_codepoint
 hb_buffer_get_replacement_codepoint
 hb_buffer_normalize_glyphs

src/hb-aat-layout-common.hh

@@ -561,15 +561,17 @@ struct StateTableDriver
 	break;
 
       /* Unsafe-to-break before this if not in state 0, as things might
-       * go differently if we start from state 0 here. */
-      if (state && buffer->idx)
+       * go differently if we start from state 0 here.
+       *
+       * Ugh.  The indexing here is ugly... */
+      if (state && buffer->backtrack_len () && buffer->idx < buffer->len)
       {
 	/* If there's no action and we're just epsilon-transitioning to state 0,
 	 * safe to break. */
 	if (c->is_actionable (this, entry) ||
 	    !(entry->newState == StateTable<EntryData>::STATE_START_OF_TEXT &&
 	      entry->flags == context_t::DontAdvance))
-	  buffer->unsafe_to_break (buffer->idx - 1, buffer->idx + 1);
+	  buffer->unsafe_to_break_from_outbuffer (buffer->backtrack_len () - 1, buffer->idx + 1);
       }
 
       /* Unsafe-to-break if end-of-text would kick in here. */

src/hb-aat-layout-kerx-table.hh

@@ -30,7 +30,6 @@
 
 #include "hb-open-type.hh"
 #include "hb-aat-layout-common.hh"
-#include "hb-aat-layout-ankr-table.hh"
 
 /*
  * kerx -- Extended Kerning
@@ -71,6 +70,14 @@ struct KerxSubTableFormat0
   //     return 0;
   //   return pairs[i].get_kerning ();
   // }
+  inline bool apply (hb_aat_apply_context_t *c) const
+  {
+    TRACE_APPLY (this);
+
+    /* TODO */
+
+    return_trace (true);
+  }
 
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
@@ -96,6 +103,15 @@ struct KerxSubTableFormat0
 
 struct KerxSubTableFormat1
 {
+  inline bool apply (hb_aat_apply_context_t *c) const
+  {
+    TRACE_APPLY (this);
+
+    /* TODO */
+
+    return_trace (true);
+  }
+
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
     TRACE_SANITIZE (this);
@@ -145,6 +161,15 @@ struct KerxSubTableFormat2
     return *v;
   }
 
+  inline bool apply (hb_aat_apply_context_t *c) const
+  {
+    TRACE_APPLY (this);
+
+    /* TODO */
+
+    return_trace (true);
+  }
+
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
     TRACE_SANITIZE (this);
@@ -172,6 +197,15 @@ struct KerxSubTableFormat2
 
 struct KerxSubTableFormat4
 {
+  inline bool apply (hb_aat_apply_context_t *c) const
+  {
+    TRACE_APPLY (this);
+
+    /* TODO */
+
+    return_trace (true);
+  }
+
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
     TRACE_SANITIZE (this);
@@ -199,6 +233,15 @@ struct KerxSubTableFormat4
 
 struct KerxSubTableFormat6
 {
+  inline bool apply (hb_aat_apply_context_t *c) const
+  {
+    TRACE_APPLY (this);
+
+    /* TODO */
+
+    return_trace (true);
+  }
+
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
     TRACE_SANITIZE (this);
@@ -221,47 +264,55 @@ struct KerxSubTableFormat6
   DEFINE_SIZE_STATIC (24);
 };
 
-enum coverage_flags_t
-{
-  COVERAGE_VERTICAL_FLAG	= 0x80u,
-  COVERAGE_CROSSSTREAM_FLAG	= 0x40u,
-  COVERAGE_VARIATION_FLAG	= 0x20u,
-  COVERAGE_PROCESS_DIRECTION	= 0x10u,
-};
-
 struct KerxTable
 {
-  inline bool apply (hb_aat_apply_context_t *c, const AAT::ankr *ankr) const
+  inline unsigned int get_size (void) const { return length; }
+  inline unsigned int get_type (void) const { return coverage & SubtableType; }
+
+  enum Coverage
   {
-    TRACE_APPLY (this);
-    /* TODO */
-    return_trace (false);
+    Vertical		= 0x80000000,	/* Set if table has vertical kerning values. */
+    CrossStream		= 0x40000000,	/* Set if table has cross-stream kerning values. */
+    Variation		= 0x20000000,	/* Set if table has variation kerning values. */
+    ProcessDirection	= 0x10000000,	/* If clear, process the glyphs forwards, that
+					 * is, from first to last in the glyph stream.
+					 * If we, process them from last to first.
+					 * This flag only applies to state-table based
+					 * 'kerx' subtables (types 1 and 4). */
+    Reserved		= 0x0FFFFF00,	/* Reserved, set to zero. */
+    SubtableType	= 0x000000FF,	/* Subtable type. */
+  };
+
+  template <typename context_t>
+  inline typename context_t::return_t dispatch (context_t *c) const
+  {
+    unsigned int subtable_type = get_type ();
+    TRACE_DISPATCH (this, subtable_type);
+    switch (subtable_type) {
+    case 0	:		return_trace (c->dispatch (u.format0));
+    case 1	:		return_trace (c->dispatch (u.format1));
+    case 2	:		return_trace (c->dispatch (u.format2));
+    case 4	:		return_trace (c->dispatch (u.format4));
+    case 6	:		return_trace (c->dispatch (u.format6));
+    default:			return_trace (c->default_return_value ());
+    }
   }
 
-  inline unsigned int get_size (void) const { return length; }
-
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
     TRACE_SANITIZE (this);
-    if (unlikely (!c->check_struct (this)))
+    if (!length.sanitize (c) ||
+	length < min_size ||
+	!c->check_range (this, length))
       return_trace (false);
 
-    switch (format) {
-    case 0: return u.format0.sanitize (c);
-    case 1: return u.format1.sanitize (c);
-    case 2: return u.format2.sanitize (c);
-    case 4: return u.format4.sanitize (c);
-    case 6: return u.format6.sanitize (c);
-    default:return_trace (false);
-    }
+    return_trace (dispatch (c));
   }
 
 protected:
   HBUINT32	length;
-  HBUINT8	coverage;
-  HBUINT16	unused;
-  HBUINT8	format;
-  HBUINT32	tupleIndex;
+  HBUINT32	coverage;
+  HBUINT32	tupleCount;
   union {
   KerxSubTableFormat0	format0;
   KerxSubTableFormat1	format1;
@@ -273,7 +324,7 @@ public:
   DEFINE_SIZE_MIN (12);
 };
 
-struct SubtableGlyphCoverageArray
+struct SubtableXXX
 {
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
@@ -289,55 +340,59 @@ struct SubtableGlyphCoverageArray
   DEFINE_SIZE_STATIC (12);
 };
 
+
+/*
+ * The 'kerx' Table
+ */
+
 struct kerx
 {
   static const hb_tag_t tableTag = HB_AAT_TAG_kerx;
 
-  inline bool apply (hb_aat_apply_context_t *c, const AAT::ankr *ankr) const
+  inline bool has_data (void) const { return version != 0; }
+
+  inline void apply (hb_aat_apply_context_t *c) const
   {
-    TRACE_APPLY (this);
-    const KerxTable &table = StructAfter<KerxTable> (*this);
-    return_trace (table.apply (c, ankr));
+    c->set_lookup_index (0);
+    const KerxTable *table = &firstTable;
+    unsigned int count = tableCount;
+    for (unsigned int i = 0; i < count; i++)
+    {
+      table->dispatch (c);
+      table = &StructAfter<KerxTable> (*table);
+    }
   }
 
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
     TRACE_SANITIZE (this);
-    if (unlikely (!(c->check_struct (this))))
-     return_trace (false);
-
-    /* TODO: Something like `morx`s ChainSubtable should be done here instead */
-    const KerxTable *table = &StructAfter<KerxTable> (*this);
-    if (unlikely (!(table->sanitize (c))))
+    if (!version.sanitize (c) || version < 2 ||
+	!tableCount.sanitize (c))
       return_trace (false);
 
-    for (unsigned int i = 0; i < nTables - 1; ++i)
+    const KerxTable *table = &firstTable;
+    unsigned int count = tableCount;
+    for (unsigned int i = 0; i < count; i++)
     {
+      if (!table->sanitize (c))
+	return_trace (false);
       table = &StructAfter<KerxTable> (*table);
-      if (unlikely (!(table->sanitize (c))))
-        return_trace (false);
     }
 
-    // If version is less than 3, we are done here; otherwise better to check footer also
-    if (version < 3)
-      return_trace (true);
-
-    // TODO: Investigate why this just work on some fonts no matter of version
-    // const SubtableGlyphCoverageArray &footer =
-    //   StructAfter<SubtableGlyphCoverageArray> (*table);
-    // return_trace (footer.sanitize (c));
-
     return_trace (true);
   }
 
   protected:
-  HBUINT16		version;
-  HBUINT16		padding;
-  HBUINT32		nTables;
-/*KerxTable tablesZ[VAR]; XXX ArrayOf??? */
-/*SubtableGlyphCoverageArray coverage_array;*/
+  HBUINT16	version;	/* The version number of the extended kerning table
+				 * (currently 2, 3, or 4). */
+  HBUINT16	unused;		/* Set to 0. */
+  HBUINT32	tableCount;	/* The number of subtables included in the extended kerning
+				 * table. */
+  KerxTable	firstTable;	/* Subtables. */
+/*subtableGlyphCoverageArray*/	/* Only if version >= 3. We don't use. */
+
   public:
-  DEFINE_SIZE_STATIC (8);
+  DEFINE_SIZE_MIN (8);
 };
 
 } /* namespace AAT */

src/hb-aat-layout-morx-table.hh

@@ -664,6 +664,8 @@ struct InsertionSubtable
 	 * This suggests that if DontAdvance is NOT set, we should move to
 	 * end+count.  If it *was*, then move to end, such that newly inserted
 	 * glyphs are now visible.
+	 *
+	 * https://github.com/harfbuzz/harfbuzz/issues/1224#issuecomment-427691417
 	 */
 	buffer->move_to ((flags & DontAdvance) ? end : end + count);
       }
@@ -940,7 +942,7 @@ struct Chain
 
 
 /*
- * The 'mort'/'morx' Tables
+ * The 'morx' Table
  */
 
 struct morx

src/hb-buffer.cc

@@ -182,7 +182,11 @@ hb_buffer_t::shift_forward (unsigned int count)
   if (idx + count > len)
   {
     /* Under memory failure we might expose this area.  At least
-     * clean it up.  Oh well... */
+     * clean it up.  Oh well...
+     *
+     * Ideally, we should at least set Default_Ignorable bits on
+     * these, as well as consistent cluster values.  But the former
+     * is layering violation... */
     memset (info + len, 0, (idx + count - len) * sizeof (info[0]));
   }
   len += count;
@@ -219,6 +223,7 @@ hb_buffer_t::reset (void)
   unicode = hb_unicode_funcs_reference (hb_unicode_funcs_get_default ());
   flags = HB_BUFFER_FLAG_DEFAULT;
   replacement = HB_BUFFER_REPLACEMENT_CODEPOINT_DEFAULT;
+  invisible = 0;
 
   clear ();
 }
@@ -398,8 +403,14 @@ hb_buffer_t::move_to (unsigned int i)
     unsigned int count = out_len - i;
 
     /* This will blow in our face if memory allocation fails later
-     * in this same lookup... */
-    if (unlikely (idx < count && !shift_forward (count + 32))) return false;
+     * in this same lookup...
+     *
+     * We used to shift with extra 32 items, instead of the 0 below.
+     * But that would leave empty slots in the buffer in case of allocation
+     * failures.  Setting to zero for now to avoid other problems (see
+     * comments in shift_forward().  This can cause O(N^2) behavior more
+     * severely than adding 32 empty slots can... */
+    if (unlikely (idx < count && !shift_forward (count + 0))) return false;
 
     assert (idx >= count);
 
@@ -665,6 +676,7 @@ DEFINE_NULL_INSTANCE (hb_buffer_t) =
   HB_BUFFER_FLAG_DEFAULT,
   HB_BUFFER_CLUSTER_LEVEL_DEFAULT,
   HB_BUFFER_REPLACEMENT_CODEPOINT_DEFAULT,
+  0, /* invisible */
   HB_BUFFER_SCRATCH_FLAG_DEFAULT,
   HB_BUFFER_MAX_LEN_DEFAULT,
   HB_BUFFER_MAX_OPS_DEFAULT,
@@ -1166,6 +1178,46 @@ hb_buffer_get_replacement_codepoint (hb_buffer_t    *buffer)
 }
 
 
+/**
+ * hb_buffer_set_invisible_glyph:
+ * @buffer: an #hb_buffer_t.
+ * @invisible: the invisible #hb_codepoint_t
+ *
+ * Sets the #hb_codepoint_t that replaces invisible characters in
+ * the shaping result.  If set to zero (default), the glyph for the
+ * U+0020 SPACE character is used.  Otherwise, this value is used
+ * verbatim.
+ *
+ * Since: REPLACEME
+ **/
+void
+hb_buffer_set_invisible_glyph (hb_buffer_t    *buffer,
+			       hb_codepoint_t  invisible)
+{
+  if (unlikely (hb_object_is_inert (buffer)))
+    return;
+
+  buffer->invisible = invisible;
+}
+
+/**
+ * hb_buffer_get_invisible_glyph:
+ * @buffer: an #hb_buffer_t.
+ *
+ * See hb_buffer_set_invisible_glyph().
+ *
+ * Return value: 
+ * The @buffer invisible #hb_codepoint_t.
+ *
+ * Since: REPLACEME
+ **/
+hb_codepoint_t
+hb_buffer_get_invisible_glyph (hb_buffer_t    *buffer)
+{
+  return buffer->invisible;
+}
+
+
 /**
  * hb_buffer_reset:
  * @buffer: an #hb_buffer_t.

src/hb-buffer.h

@@ -341,6 +341,13 @@ hb_buffer_set_replacement_codepoint (hb_buffer_t    *buffer,
 HB_EXTERN hb_codepoint_t
 hb_buffer_get_replacement_codepoint (hb_buffer_t    *buffer);
 
+HB_EXTERN void
+hb_buffer_set_invisible_glyph (hb_buffer_t    *buffer,
+			       hb_codepoint_t  invisible);
+
+HB_EXTERN hb_codepoint_t
+hb_buffer_get_invisible_glyph (hb_buffer_t    *buffer);
+
 
 HB_EXTERN void
 hb_buffer_reset (hb_buffer_t *buffer);

src/hb-buffer.hh

@@ -93,6 +93,7 @@ struct hb_buffer_t
   hb_buffer_flags_t flags; /* BOT / EOT / etc. */
   hb_buffer_cluster_level_t cluster_level;
   hb_codepoint_t replacement; /* U+FFFD or something else. */
+  hb_codepoint_t invisible; /* 0 or something else. */
   hb_buffer_scratch_flags_t scratch_flags; /* Have space-fallback, etc. */
   unsigned int max_len; /* Maximum allowed len. */
   int max_ops; /* Maximum allowed operations. */

src/hb-coretext.cc

@@ -210,7 +210,7 @@ create_ct_font (CGFontRef cg_font, CGFloat font_size)
   }
 
   CFURLRef original_url = nullptr;
-#if TARGET_OS_MAC && MAC_OS_X_VERSION_MIN_REQUIRED < 1060
+#if TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED < 1060
   ATSFontRef atsFont;
   FSRef fsref;
   OSStatus status;
@@ -240,7 +240,7 @@ create_ct_font (CGFontRef cg_font, CGFloat font_size)
        * process in Blink. This can be detected by the new file URL location
        * that the newly found font points to. */
       CFURLRef new_url = nullptr;
-#if TARGET_OS_MAC && MAC_OS_X_VERSION_MIN_REQUIRED < 1060
+#if TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED < 1060
       atsFont = CTFontGetPlatformFont (new_ct_font, NULL);
       status = ATSFontGetFileReference (atsFont, &fsref);
       if (status == noErr)
@@ -768,7 +768,7 @@ _hb_coretext_shape (hb_shape_plan_t    *shape_plan,
       } else {
         active_feature_t *feature = active_features.find (&event->feature);
 	if (feature)
-	  active_features.remove (feature - active_features.arrayZ);
+	  active_features.remove (feature - active_features.arrayZ());
       }
     }
   }

src/hb-ot-font.cc

@@ -75,7 +75,7 @@ hb_ot_get_glyph_h_advances (hb_font_t* font, void* font_data,
 			    void *user_data HB_UNUSED)
 {
   const hb_ot_face_data_t *ot_face = (const hb_ot_face_data_t *) font_data;
-  const OT::hmtx_accelerator_t &hmtx = *ot_face->hmtx.get_relaxed ();
+  const OT::hmtx_accelerator_t &hmtx = *ot_face->hmtx.get ();
 
   for (unsigned int i = 0; i < count; i++)
   {
@@ -95,7 +95,7 @@ hb_ot_get_glyph_v_advances (hb_font_t* font, void* font_data,
 			    void *user_data HB_UNUSED)
 {
   const hb_ot_face_data_t *ot_face = (const hb_ot_face_data_t *) font_data;
-  const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx.get_relaxed ();
+  const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx.get ();
 
   for (unsigned int i = 0; i < count; i++)
   {
@@ -170,11 +170,12 @@ hb_ot_get_font_h_extents (hb_font_t *font,
 			  void *user_data HB_UNUSED)
 {
   const hb_ot_face_data_t *ot_face = (const hb_ot_face_data_t *) font_data;
-  metrics->ascender = font->em_scale_y (ot_face->hmtx.get_relaxed ()->ascender);
-  metrics->descender = font->em_scale_y (ot_face->hmtx.get_relaxed ()->descender);
-  metrics->line_gap = font->em_scale_y (ot_face->hmtx.get_relaxed ()->line_gap);
+  const OT::hmtx_accelerator_t &hmtx = *ot_face->hmtx.get ();
+  metrics->ascender = font->em_scale_y (hmtx.ascender);
+  metrics->descender = font->em_scale_y (hmtx.descender);
+  metrics->line_gap = font->em_scale_y (hmtx.line_gap);
   // TODO Hook up variations.
-  return ot_face->hmtx.get_relaxed ()->has_font_extents;
+  return hmtx.has_font_extents;
 }
 
 static hb_bool_t
@@ -184,18 +185,19 @@ hb_ot_get_font_v_extents (hb_font_t *font,
 			  void *user_data HB_UNUSED)
 {
   const hb_ot_face_data_t *ot_face = (const hb_ot_face_data_t *) font_data;
-  metrics->ascender = font->em_scale_x (ot_face->vmtx.get_relaxed ()->ascender);
-  metrics->descender = font->em_scale_x (ot_face->vmtx.get_relaxed ()->descender);
-  metrics->line_gap = font->em_scale_x (ot_face->vmtx.get_relaxed ()->line_gap);
+  const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx.get ();
+  metrics->ascender = font->em_scale_x (vmtx.ascender);
+  metrics->descender = font->em_scale_x (vmtx.descender);
+  metrics->line_gap = font->em_scale_x (vmtx.line_gap);
   // TODO Hook up variations.
-  return ot_face->vmtx.get_relaxed ()->has_font_extents;
+  return vmtx.has_font_extents;
 }
 
 #ifdef HB_USE_ATEXIT
 static void free_static_ot_funcs (void);
 #endif
 
-static struct hb_ot_face_funcs_lazy_loader_t : hb_font_funcs_lazy_loader_t<hb_ot_face_funcs_lazy_loader_t>
+static struct hb_ot_font_funcs_lazy_loader_t : hb_font_funcs_lazy_loader_t<hb_ot_font_funcs_lazy_loader_t>
 {
   static inline hb_font_funcs_t *create (void)
   {
@@ -252,10 +254,8 @@ hb_ot_font_set_funcs (hb_font_t *font)
   if (unlikely (!hb_ot_shaper_face_data_ensure (font->face))) return;
   hb_ot_face_data_t *ot_face = hb_ot_face_data (font->face);
 
-  /* Load them lazies.  We access them with get_relaxed() for performance. */
+  /* Load them lazy.  We access it with get_relaxed() for performance. */
   ot_face->cmap.get ();
-  ot_face->hmtx.get ();
-  ot_face->vmtx.get ();
 
   hb_font_set_funcs (font,
 		     _hb_ot_get_font_funcs (),

src/hb-ot-shape.cc

@@ -537,34 +537,24 @@ hb_ot_hide_default_ignorables (hb_ot_shape_context_t *c)
   unsigned int count = buffer->len;
   hb_glyph_info_t *info = buffer->info;
   hb_glyph_position_t *pos = buffer->pos;
-  unsigned int i = 0;
-  for (i = 0; i < count; i++)
-  {
-    if (unlikely (_hb_glyph_info_is_default_ignorable (&info[i])))
-      break;
-  }
-
-  /* No default-ignorables found; return. */
-  if (i == count)
-    return;
 
-  hb_codepoint_t space;
+  hb_codepoint_t invisible = c->buffer->invisible;
   if (!(buffer->flags & HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES) &&
-      c->font->get_nominal_glyph (' ', &space))
+      (invisible || c->font->get_nominal_glyph (' ', &invisible)))
   {
-    /* Replace default-ignorables with a zero-advance space glyph. */
-    for (/*continue*/; i < count; i++)
+    /* Replace default-ignorables with a zero-advance invisible glyph. */
+    for (unsigned int i = 0; i < count; i++)
     {
       if (_hb_glyph_info_is_default_ignorable (&info[i]))
-	info[i].codepoint = space;
+	info[i].codepoint = invisible;
     }
   }
   else
   {
     /* Merge clusters and delete default-ignorables.
      * NOTE! We can't use out-buffer as we have positioning data. */
-    unsigned int j = i;
-    for (; i < count; i++)
+    unsigned int j = 0;
+    for (unsigned int i = 0; i < count; i++)
     {
       if (_hb_glyph_info_is_default_ignorable (&info[i]))
       {

src/hb-uniscribe.cc

@@ -717,7 +717,7 @@ _hb_uniscribe_shape (hb_shape_plan_t    *shape_plan,
       {
         active_feature_t *feature = active_features.find (&event->feature);
 	if (feature)
-	  active_features.remove (feature - active_features.arrayZ);
+	  active_features.remove (feature - active_features.arrayZ());
       }
     }
 
@@ -728,7 +728,7 @@ _hb_uniscribe_shape (hb_shape_plan_t    *shape_plan,
     for (unsigned int i = 0; i < range_records.len; i++)
     {
       range_record_t *range = &range_records[i];
-      range->props.potfRecords = feature_records.arrayZ + reinterpret_cast<uintptr_t> (range->props.potfRecords);
+      range->props.potfRecords = feature_records.arrayZ() + reinterpret_cast<uintptr_t> (range->props.potfRecords);
     }
   }
 
@@ -902,8 +902,8 @@ retry:
 				     &items[i].a,
 				     script_tags[i],
 				     language_tag,
-				     range_char_counts.arrayZ,
-				     range_properties.arrayZ,
+				     range_char_counts.arrayZ(),
+				     range_properties.arrayZ(),
 				     range_properties.len,
 				     pchars + chars_offset,
 				     item_chars_len,
@@ -943,8 +943,8 @@ retry:
 				     &items[i].a,
 				     script_tags[i],
 				     language_tag,
-				     range_char_counts.arrayZ,
-				     range_properties.arrayZ,
+				     range_char_counts.arrayZ(),
+				     range_properties.arrayZ(),
 				     range_properties.len,
 				     pchars + chars_offset,
 				     log_clusters + chars_offset,

test/api/test-multithread.c

@@ -135,7 +135,7 @@ main (int argc, char **argv)
   gchar *default_path = g_strdup (font_path);
 #endif
 
-  char *path = argc > 1 ? argv[1] : (char *) default_path;
+  char *path = argc > 1 && *argv[1] ? argv[1] : (char *) default_path;
   if (argc > 2)
     num_threads = atoi (argv[2]);
   if (argc > 3)
@@ -143,8 +143,8 @@ main (int argc, char **argv)
   if (argc > 4)
     text = argv[4];
 
-  // Dummy call to alleviate _guess_segment_properties thread safety-ness
-  // https://github.com/harfbuzz/harfbuzz/issues/1191
+  /* Dummy call to alleviate _guess_segment_properties thread safety-ness
+   * https://github.com/harfbuzz/harfbuzz/issues/1191 */
   hb_language_get_default ();
 
   hb_blob_t *blob = hb_blob_create_from_file (path);

test/shaping/data/text-rendering-tests/Makefile.sources

@@ -56,7 +56,11 @@ TESTS = \
 	tests/MORX-33.tests \
 	tests/MORX-34.tests \
 	tests/MORX-36.tests \
+	tests/MORX-37.tests \
+	tests/MORX-38.tests \
+	tests/MORX-39.tests \
 	tests/MORX-3.tests \
+	tests/MORX-40.tests \
 	tests/MORX-4.tests \
 	tests/MORX-5.tests \
 	tests/MORX-6.tests \
@@ -68,8 +72,8 @@ TESTS = \
 	$(NULL)
 
 DISBALED_TESTS = \
-	tests/MORX-35.tests \
 	tests/CMAP-3.tests \
+	tests/MORX-35.tests \
 	tests/SHARAN-1.tests \
 	tests/SHBALI-1.tests \
 	tests/SHBALI-2.tests \

test/shaping/data/text-rendering-tests/tests/MORX-35.tests

-../fonts/TestMORXThirtyfive.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0041:[A|B@639,0|C@1265,0]
-../fonts/TestMORXThirtyfive.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0058,U+0041,U+0059:[X|A@586,0|B@1225,0|C@1851,0|E@2447,0|Y@3003,0]
+../fonts/TestMORXThirtyfive.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0041:[A|B@639,0|E@1265,0|C@1821,0|E@2417,0]
+../fonts/TestMORXThirtyfive.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0058,U+0041,U+0059:[X|A@586,0|B@1225,0|E@1851,0|C@2407,0|E@3003,0|Y@3559,0]

test/shaping/data/text-rendering-tests/tests/MORX-37.tests

+../fonts/TestMORXThirtyseven.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0041,U+0042:[A.alt|B.alt@1000,0]
+../fonts/TestMORXThirtyseven.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0042,U+0041:[B|A@650,0]
+../fonts/TestMORXThirtyseven.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+05D0,U+05D1:[uni05D1|uni05D0@542,0]
+../fonts/TestMORXThirtyseven.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+05D1,U+05D0:[uni05D0.alt|uni05D1.alt@1000,0]

test/shaping/data/text-rendering-tests/tests/MORX-38.tests

+../fonts/TestMORXThirtyeight.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0041,U+0042:[A.alt|B.alt@1000,0]
+../fonts/TestMORXThirtyeight.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0042,U+0041:[B|A@650,0]
+../fonts/TestMORXThirtyeight.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+05D0,U+05D1:[uni05D1.alt|uni05D0.alt@1000,0]
+../fonts/TestMORXThirtyeight.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+05D1,U+05D0:[uni05D0|uni05D1@606,0]

test/shaping/data/text-rendering-tests/tests/MORX-39.tests

+../fonts/TestMORXThirtynine.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0041,U+0042:[A|B@639,0]
+../fonts/TestMORXThirtynine.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0042,U+0041:[B.alt|A.alt@1000,0]
+../fonts/TestMORXThirtynine.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+05D0,U+05D1:[uni05D1.alt|uni05D0.alt@1000,0]
+../fonts/TestMORXThirtynine.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+05D1,U+05D0:[uni05D0|uni05D1@606,0]

test/shaping/data/text-rendering-tests/tests/MORX-40.tests

+../fonts/TestMORXForty.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0041,U+0042:[A|B@639,0]
+../fonts/TestMORXForty.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+0042,U+0041:[B.alt|A.alt@1000,0]
+../fonts/TestMORXForty.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+05D0,U+05D1:[uni05D1|uni05D0@542,0]
+../fonts/TestMORXForty.ttf:--font-size=1000 --ned --remove-default-ignorables --font-funcs=ft:U+05D1,U+05D0:[uni05D0.alt|uni05D1.alt@1000,0]

test/shaping/run-tests.py

@@ -2,16 +2,16 @@
 
 from __future__ import print_function, division, absolute_import
 
-import sys, os, subprocess
+import sys, os, subprocess, tempfile
 
 
 def cmd(command):
-	p = subprocess.Popen (
-		command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-	p.wait ()
-	print (p.stderr.read ().decode ("utf-8").strip ()) # file=sys.stderr
-	return p.stdout.read ().decode ("utf-8").strip (), p.returncode
-
+	# https://stackoverflow.com/a/4408409
+	with tempfile.TemporaryFile() as tempf:
+		p = subprocess.Popen (command, stdout=tempf, stderr=sys.stdout)
+		p.wait ()
+		tempf.seek(0)
+		return tempf.read().decode ("utf-8").strip (), p.returncode
 
 args = sys.argv[1:]
 if not args or sys.argv[1].find('hb-shape') == -1 or not os.path.exists (sys.argv[1]):

util/options.cc

@@ -415,6 +415,7 @@ shape_options_t::add_options (option_parser_t *parser)
     {"eot",		0, 0, G_OPTION_ARG_NONE,	&this->eot,			"Treat text as end-of-paragraph",	nullptr},
     {"preserve-default-ignorables",0, 0, G_OPTION_ARG_NONE,	&this->preserve_default_ignorables,	"Preserve Default-Ignorable characters",	nullptr},
     {"remove-default-ignorables",0, 0, G_OPTION_ARG_NONE,	&this->remove_default_ignorables,	"Remove Default-Ignorable characters",	nullptr},
+    {"invisible-glyph",	0, 0, G_OPTION_ARG_INT,		&this->invisible_glyph,		"Glyph value to replace Default-Ignorables with",	nullptr},
     {"utf8-clusters",	0, 0, G_OPTION_ARG_NONE,	&this->utf8_clusters,		"Use UTF8 byte indices, not char indices",	nullptr},
     {"cluster-level",	0, 0, G_OPTION_ARG_INT,		&this->cluster_level,		"Cluster merging level (default: 0)",	"0/1/2"},
     {"normalize-glyphs",0, 0, G_OPTION_ARG_NONE,	&this->normalize_glyphs,	"Rearrange glyph clusters in nominal order",	nullptr},

util/options.hh

@@ -155,6 +155,7 @@ struct shape_options_t : option_group_t
     num_features = 0;
     shapers = nullptr;
     utf8_clusters = false;
+    invisible_glyph = 0;
     cluster_level = HB_BUFFER_CLUSTER_LEVEL_DEFAULT;
     normalize_glyphs = false;
     verify = false;
@@ -185,6 +186,7 @@ struct shape_options_t : option_group_t
 				  (preserve_default_ignorables ? HB_BUFFER_FLAG_PRESERVE_DEFAULT_IGNORABLES : 0) |
 				  (remove_default_ignorables ? HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES : 0) |
 				  0));
+    hb_buffer_set_invisible_glyph (buffer, invisible_glyph);
     hb_buffer_set_cluster_level (buffer, cluster_level);
     hb_buffer_guess_segment_properties (buffer);
   }
@@ -435,6 +437,7 @@ struct shape_options_t : option_group_t
   unsigned int num_features;
   char **shapers;
   hb_bool_t utf8_clusters;
+  hb_codepoint_t invisible_glyph;
   hb_buffer_cluster_level_t cluster_level;
   hb_bool_t normalize_glyphs;
   hb_bool_t verify;