Limit how much recursion GSUB/GPOS does

This only counts recursions right now.  Good start.

Hopefully...
Fixes https://github.com/behdad/harfbuzz/issues/429

src/Makefile.am

@@ -138,9 +138,12 @@ FUZZING_CPPFLAGS= \
 	-DHB_NDEBUG \
 	-DHB_MAX_NESTING_LEVEL=3 \
 	-DHB_SANITIZE_MAX_EDITS=3 \
-	-DHB_BUFFER_MAX_EXPANSION_FACTOR=3 \
+	-DHB_BUFFER_MAX_LEN_FACTOR=3 \
 	-DHB_BUFFER_MAX_LEN_MIN=8 \
 	-DHB_BUFFER_MAX_LEN_DEFAULT=128 \
+	-DHB_BUFFER_MAX_OPS_FACTOR=8 \
+	-DHB_BUFFER_MAX_OPS_MIN=64 \
+	-DHB_BUFFER_MAX_OPS_DEFAULT=1024 \
 	$(NULL)
 EXTRA_LTLIBRARIES = libharfbuzz-fuzzing.la
 libharfbuzz_fuzzing_la_LINK = $(libharfbuzz_la_LINK)

src/hb-buffer-private.hh

@@ -35,8 +35,8 @@
 #include "hb-unicode-private.hh"
 
 
-#ifndef HB_BUFFER_MAX_EXPANSION_FACTOR
-#define HB_BUFFER_MAX_EXPANSION_FACTOR 32
+#ifndef HB_BUFFER_MAX_LEN_FACTOR
+#define HB_BUFFER_MAX_LEN_FACTOR 32
 #endif
 #ifndef HB_BUFFER_MAX_LEN_MIN
 #define HB_BUFFER_MAX_LEN_MIN 8192
@@ -45,6 +45,16 @@
 #define HB_BUFFER_MAX_LEN_DEFAULT 0x3FFFFFFF /* Shaping more than a billion chars? Let us know! */
 #endif
 
+#ifndef HB_BUFFER_MAX_OPS_FACTOR
+#define HB_BUFFER_MAX_OPS_FACTOR 64
+#endif
+#ifndef HB_BUFFER_MAX_OPS_MIN
+#define HB_BUFFER_MAX_OPS_MIN 1024
+#endif
+#ifndef HB_BUFFER_MAX_OPS_DEFAULT
+#define HB_BUFFER_MAX_OPS_DEFAULT 0x1FFFFFFF /* Shaping more than a billion operations? Let us know! */
+#endif
+
 static_assert ((sizeof (hb_glyph_info_t) == 20), "");
 static_assert ((sizeof (hb_glyph_info_t) == sizeof (hb_glyph_position_t)), "");
 
@@ -84,6 +94,7 @@ struct hb_buffer_t {
   hb_codepoint_t replacement; /* U+FFFD or something else. */
   hb_buffer_scratch_flags_t scratch_flags; /* Have space-flallback, etc. */
   unsigned int max_len; /* Maximum allowed len. */
+  int max_ops; /* Maximum allowed operations. */
 
   /* Buffer contents */
   hb_buffer_content_type_t content_type;

src/hb-buffer.cc

@@ -722,6 +722,7 @@ hb_buffer_create (void)
     return hb_buffer_get_empty ();
 
   buffer->max_len = HB_BUFFER_MAX_LEN_DEFAULT;
+  buffer->max_ops = HB_BUFFER_MAX_OPS_DEFAULT;
 
   buffer->reset ();
 
@@ -749,6 +750,7 @@ hb_buffer_get_empty (void)
     HB_BUFFER_REPLACEMENT_CODEPOINT_DEFAULT,
     HB_BUFFER_SCRATCH_FLAG_DEFAULT,
     HB_BUFFER_MAX_LEN_DEFAULT,
+    HB_BUFFER_MAX_OPS_DEFAULT,
 
     HB_BUFFER_CONTENT_TYPE_INVALID,
     HB_SEGMENT_PROPERTIES_DEFAULT,

src/hb-ot-layout-gsubgpos-private.hh

@@ -413,7 +413,7 @@ struct hb_apply_context_t :
   bool stop_sublookup_iteration (return_t r) const { return r; }
   return_t recurse (unsigned int lookup_index)
   {
-    if (unlikely (nesting_level_left == 0 || !recurse_func))
+    if (unlikely (nesting_level_left == 0 || !recurse_func || buffer->max_ops-- <= 0))
       return default_return_value ();
 
     nesting_level_left--;
@@ -1005,6 +1005,9 @@ static inline bool apply_lookup (hb_apply_context_t *c,
     if (unlikely (!buffer->move_to (match_positions[idx])))
       break;
 
+    if (unlikely (buffer->max_ops <= 0))
+      break;
+
     unsigned int orig_len = buffer->backtrack_len () + buffer->lookahead_len ();
     if (!c->recurse (lookupRecord[i].lookupListIndex))
       continue;

src/hb-ot-shape.cc

@@ -817,11 +817,16 @@ hb_ot_shape_internal (hb_ot_shape_context_t *c)
 {
   c->buffer->deallocate_var_all ();
   c->buffer->scratch_flags = HB_BUFFER_SCRATCH_FLAG_DEFAULT;
-  if (likely (!_hb_unsigned_int_mul_overflows (c->buffer->len, HB_BUFFER_MAX_EXPANSION_FACTOR)))
+  if (likely (!_hb_unsigned_int_mul_overflows (c->buffer->len, HB_BUFFER_MAX_LEN_FACTOR)))
   {
-    c->buffer->max_len = MAX (c->buffer->len * HB_BUFFER_MAX_EXPANSION_FACTOR,
+    c->buffer->max_len = MAX (c->buffer->len * HB_BUFFER_MAX_LEN_FACTOR,
 			      (unsigned) HB_BUFFER_MAX_LEN_MIN);
   }
+  if (likely (!_hb_unsigned_int_mul_overflows (c->buffer->len, HB_BUFFER_MAX_OPS_FACTOR)))
+  {
+    c->buffer->max_ops = MAX (c->buffer->len * HB_BUFFER_MAX_OPS_FACTOR,
+			      (unsigned) HB_BUFFER_MAX_OPS_MIN);
+  }
 
   bool disable_otl = c->plan->shaper->disable_otl && c->plan->shaper->disable_otl (c->plan);
   //c->fallback_substitute     = disable_otl || !hb_ot_layout_has_substitution (c->face);
@@ -861,6 +866,7 @@ hb_ot_shape_internal (hb_ot_shape_context_t *c)
   c->buffer->props.direction = c->target_direction;
 
   c->buffer->max_len = HB_BUFFER_MAX_LEN_DEFAULT;
+  c->buffer->max_ops = HB_BUFFER_MAX_OPS_DEFAULT;
   c->buffer->deallocate_var_all ();
 }
 

test/shaping/tests/fuzzed.tests

@@ -16,4 +16,4 @@ fonts/sha1sum/a69118c2c2ada48ff803d9149daa54c9ebdae30e.ttf:--font-funcs=ot:U+004
 fonts/sha1sum/b6acef662e0beb8d5fcf5b61c6b0ca69537b7402.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
 fonts/sha1sum/e88c339237f52d21e01c55f01b9c1b4cc14a0467.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
 fonts/sha1sum/243798dd281c1c77c065958e1ff467420faa9bde.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
-fonts/sha1sum/dd9f0c7c7c36f75a18be0cab1cddf8f3ab0f366b.ttf:--font-funcs=ot --no-positions --no-clusters --no-glyph-names:U+0041:[0|0|0|2|0|0|2|0|0|2|0|0|0|2|0|0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|0|2|0|0|0|2|0|0|0|2|0|0|2|0|0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|0|2|0|0|2|0|0|2|0|0|2|0]
+fonts/sha1sum/dd9f0c7c7c36f75a18be0cab1cddf8f3ab0f366b.ttf:--font-funcs=ot --no-positions --no-clusters --no-glyph-names:U+0041:[0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|0|2|0|0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|0|2|0|0|2|0|0|2|0|0|2|0]