Try to better handle OOM situations

Fixes assert fail in https://github.com/behdad/harfbuzz/issues/161
with libharfbuzz-fuzzing.

src/hb-ot-layout-gsubgpos-private.hh

@@ -842,7 +842,7 @@ static inline bool ligate_input (hb_apply_context_t *c,
 
   for (unsigned int i = 1; i < count; i++)
   {
-    while (buffer->idx < match_positions[i])
+    while (buffer->idx < match_positions[i] && !buffer->in_error)
     {
       if (!is_mark_ligature) {
 	unsigned int new_lig_comp = components_so_far - last_num_components +

src/hb-ot-layout.cc

@@ -890,7 +890,7 @@ apply_forward (OT::hb_apply_context_t *c,
 {
   bool ret = false;
   hb_buffer_t *buffer = c->buffer;
-  while (buffer->idx < buffer->len)
+  while (buffer->idx < buffer->len && !buffer->in_error)
   {
     if (accel.may_have (buffer->cur().codepoint) &&
 	(buffer->cur().mask & c->lookup_mask) &&

src/hb-ot-shape-complex-hangul.cc

@@ -188,7 +188,7 @@ preprocess_text_hangul (const hb_ot_shape_plan_t *plan,
 				    */
   unsigned int count = buffer->len;
 
-  for (buffer->idx = 0; buffer->idx < count;)
+  for (buffer->idx = 0; buffer->idx < count && !buffer->in_error;)
   {
     hb_codepoint_t u = buffer->cur().codepoint;
 

src/hb-ot-shape-complex-indic.cc

@@ -1243,7 +1243,7 @@ insert_dotted_circles (const hb_ot_shape_plan_t *plan HB_UNUSED,
 
   buffer->idx = 0;
   unsigned int last_syllable = 0;
-  while (buffer->idx < buffer->len)
+  while (buffer->idx < buffer->len && !buffer->in_error)
   {
     unsigned int syllable = buffer->cur().syllable();
     syllable_type_t syllable_type = (syllable_type_t) (syllable & 0x0F);

src/hb-ot-shape-complex-thai.cc

@@ -315,7 +315,7 @@ preprocess_text_thai (const hb_ot_shape_plan_t *plan,
 
   buffer->clear_output ();
   unsigned int count = buffer->len;
-  for (buffer->idx = 0; buffer->idx < count;)
+  for (buffer->idx = 0; buffer->idx < count && !buffer->in_error;)
   {
     hb_codepoint_t u = buffer->cur().codepoint;
     if (likely (!IS_SARA_AM (u))) {

src/hb-ot-shape-complex-use.cc

@@ -490,11 +490,6 @@ insert_dotted_circles (const hb_ot_shape_plan_t *plan HB_UNUSED,
   if (likely (!has_broken_syllables))
     return;
 
-
-  hb_codepoint_t dottedcircle_glyph;
-  if (!font->get_glyph (0x25CCu, 0, &dottedcircle_glyph))
-    return;
-
   hb_glyph_info_t dottedcircle = {0};
   if (!font->get_glyph (0x25CCu, 0, &dottedcircle.codepoint))
     return;
@@ -503,9 +498,8 @@ insert_dotted_circles (const hb_ot_shape_plan_t *plan HB_UNUSED,
   buffer->clear_output ();
 
   buffer->idx = 0;
-
   unsigned int last_syllable = 0;
-  while (buffer->idx < buffer->len)
+  while (buffer->idx < buffer->len && !buffer->in_error)
   {
     unsigned int syllable = buffer->cur().syllable();
     syllable_type_t syllable_type = (syllable_type_t) (syllable & 0x0F);

src/hb-ot-shape-normalize.cc

@@ -218,7 +218,7 @@ handle_variation_selector_cluster (const hb_ot_shape_normalize_context_t *c, uns
   /* TODO Currently if there's a variation-selector we give-up, it's just too hard. */
   hb_buffer_t * const buffer = c->buffer;
   hb_font_t * const font = c->font;
-  for (; buffer->idx < end - 1;) {
+  for (; buffer->idx < end - 1 && !buffer->in_error;) {
     if (unlikely (buffer->unicode->is_variation_selector (buffer->cur(+1).codepoint))) {
       /* The next two lines are some ugly lines... But work. */
       if (font->get_glyph (buffer->cur().codepoint, buffer->cur(+1).codepoint, &buffer->cur().glyph_index()))
@@ -254,13 +254,13 @@ static inline void
 decompose_multi_char_cluster (const hb_ot_shape_normalize_context_t *c, unsigned int end, bool short_circuit)
 {
   hb_buffer_t * const buffer = c->buffer;
-  for (unsigned int i = buffer->idx; i < end; i++)
+  for (unsigned int i = buffer->idx; i < end && !buffer->in_error; i++)
     if (unlikely (buffer->unicode->is_variation_selector (buffer->info[i].codepoint))) {
       handle_variation_selector_cluster (c, end, short_circuit);
       return;
     }
 
-  while (buffer->idx < end)
+  while (buffer->idx < end && !buffer->in_error)
     decompose_current_character (c, short_circuit);
 }
 
@@ -320,7 +320,7 @@ _hb_ot_shape_normalize (const hb_ot_shape_plan_t *plan,
 
   buffer->clear_output ();
   count = buffer->len;
-  for (buffer->idx = 0; buffer->idx < count;)
+  for (buffer->idx = 0; buffer->idx < count && !buffer->in_error;)
   {
     unsigned int end;
     for (end = buffer->idx + 1; end < count; end++)
@@ -370,7 +370,7 @@ _hb_ot_shape_normalize (const hb_ot_shape_plan_t *plan,
   count = buffer->len;
   unsigned int starter = 0;
   buffer->next_glyph ();
-  while (buffer->idx < count)
+  while (buffer->idx < count && !buffer->in_error)
   {
     hb_codepoint_t composed, glyph;
     if (/* We don't try to compose a non-mark character with it's preceding starter.

src/hb-ot-shape.cc

@@ -254,7 +254,7 @@ hb_insert_dotted_circle (hb_buffer_t *buffer, hb_font_t *font)
   info.cluster = buffer->cur().cluster;
   info.mask = buffer->cur().mask;
   buffer->output_info (info);
-  while (buffer->idx < buffer->len)
+  while (buffer->idx < buffer->len && !buffer->in_error)
     buffer->next_glyph ();
 
   buffer->swap_buffers ();