minimize use of assert: removed or changed to error handling (#1467)

src/hb-ot-cff-common.hh

@@ -51,7 +51,7 @@ inline unsigned int calcOffSize(unsigned int dataSize)
     size++;
     offset >>= 8;
   }
-  assert (size <= 4);
+  /* format does not support size > 4; caller should handle it as an error */
   return size;
 }
 

src/hb-ot-cff1-table.hh

@@ -211,7 +211,8 @@ struct Encoding {
 	hb_codepoint_t code = code_ranges[i].code;
 	for (int left = (int)code_ranges[i].glyph; left >= 0; left--)
 	  fmt0->codes[glyph++].set (code++);
-	assert ((glyph <= 0x100) && (code <= 0x100));
+	if (unlikely (!((glyph <= 0x100) && (code <= 0x100))))
+	  return_trace (false);
       }
     }
     else
@@ -221,7 +222,8 @@ struct Encoding {
       fmt1->nRanges.set (code_ranges.len);
       for (unsigned int i = 0; i < code_ranges.len; i++)
       {
-	assert ((code_ranges[i].code <= 0xFF) && (code_ranges[i].glyph <= 0xFF));
+	if (unlikely (!((code_ranges[i].code <= 0xFF) && (code_ranges[i].glyph <= 0xFF))))
+	  return_trace (false);
 	fmt1->ranges[i].first.set (code_ranges[i].code);
 	fmt1->ranges[i].nLeft.set (code_ranges[i].glyph);
       }
@@ -490,7 +492,8 @@ struct Charset {
       if (unlikely (fmt1 == nullptr)) return_trace (false);
       for (unsigned int i = 0; i < sid_ranges.len; i++)
       {
-	assert (sid_ranges[i].glyph <= 0xFF);
+      	if (unlikely (!(sid_ranges[i].glyph <= 0xFF)))
+	  return_trace (false);
 	fmt1->ranges[i].first.set (sid_ranges[i].code);
 	fmt1->ranges[i].nLeft.set (sid_ranges[i].glyph);
       }
@@ -501,7 +504,8 @@ struct Charset {
       if (unlikely (fmt2 == nullptr)) return_trace (false);
       for (unsigned int i = 0; i < sid_ranges.len; i++)
       {
-	assert (sid_ranges[i].glyph <= 0xFFFF);
+      	if (unlikely (!(sid_ranges[i].glyph <= 0xFFFF)))
+	  return_trace (false);
 	fmt2->ranges[i].first.set (sid_ranges[i].code);
 	fmt2->ranges[i].nLeft.set (sid_ranges[i].glyph);
       }

src/hb-subset-cff-common.cc

@@ -100,8 +100,9 @@ hb_plan_subset_cff_fdselect (const hb_vector_t<hb_codepoint_t> &glyphs,
       hb_codepoint_t  fd = CFF_UNDEF_CODE;
       while (set->next (&fd))
 	fdmap.add (fd);
-      assert (fdmap.get_count () == subset_fd_count);
       hb_set_destroy (set);
+      if (unlikely (fdmap.get_count () != subset_fd_count))
+      	return false;
     }
 
     /* update each font dict index stored as "code" in fdselect_ranges */
@@ -112,7 +113,8 @@ hb_plan_subset_cff_fdselect (const hb_vector_t<hb_codepoint_t> &glyphs,
   /* determine which FDSelect format is most compact */
   if (subset_fd_count > 0xFF)
   {
-    assert (src.format == 4);
+    if (unlikely (src.format != 4))
+      return false;
     subset_fdselect_format = 4;
     subset_fdselect_size = FDSelect::min_size + FDSelect4::min_size + FDSelect4_Range::static_size * num_ranges + HBUINT32::static_size;
   }

src/hb-subset-cff-common.hh

@@ -578,7 +578,6 @@ struct SubrRemap : Remap
   inline int biased_num (unsigned int old_num) const
   {
     hb_codepoint_t new_num = (*this)[old_num];
-    assert (new_num != CFF_UNDEF_CODE);
     return (int)new_num - bias;
   }
 

src/hb-subset-cff1.cc

@@ -175,11 +175,12 @@ struct CFF1TopDict_OpSerializer : CFFTopDict_OpSerializer<CFF1TopDictVal>
 	  OpStr supp_op;
 	  supp_op.op = op;
 	  supp_op.str.str = opstr.str.str + opstr.last_arg_offset;
-	  assert (opstr.str.len >= opstr.last_arg_offset + 3);
+	  if ( unlikely (!(opstr.str.len >= opstr.last_arg_offset + 3)))
+	    return_trace (false);
 	  supp_op.str.len = opstr.str.len - opstr.last_arg_offset;
-	return_trace (UnsizedByteStr::serialize_int2 (c, mod.nameSIDs[NameDictValues::registry]) &&
-		      UnsizedByteStr::serialize_int2 (c, mod.nameSIDs[NameDictValues::ordering]) &&
-		      copy_opstr (c, supp_op));
+	  return_trace (UnsizedByteStr::serialize_int2 (c, mod.nameSIDs[NameDictValues::registry]) &&
+			UnsizedByteStr::serialize_int2 (c, mod.nameSIDs[NameDictValues::ordering]) &&
+			copy_opstr (c, supp_op));
 	}
       default:
 	return_trace (CFFTopDict_OpSerializer<CFF1TopDictVal>::serialize (c, opstr, mod.offsets));
@@ -644,6 +645,8 @@ struct cff_subset_plan {
       CFF1TopDict_OpSerializer topSzr;
       unsigned int topDictSize = TopDict::calculate_serialized_size (topdict_mod, topSzr);
       offsets.topDictInfo.offSize = calcOffSize(topDictSize);
+      if (unlikely (offsets.topDictInfo.offSize > 4))
+      	return false;
       final_size += CFF1IndexOf<TopDict>::calculate_serialized_size<CFF1TopDictValuesMod>
 						(offsets.topDictInfo.offSize,
 						 &topdict_mod, 1, topdict_sizes, topSzr);
@@ -670,7 +673,8 @@ struct cff_subset_plan {
       /* SIDs for name strings in dicts are added before glyph names so they fit in 16-bit int range */
       if (unlikely (!collect_sids_in_dicts (acc)))
 	return false;
-      assert (sidmap.get_count () <= 0x8000);
+      if (unlikely (sidmap.get_count () > 0x8000))	/* assumption: a dict won't reference that many strings */
+      	return false;
       if (subset_charset)
 	offsets.charsetInfo.size = plan_subset_charset (acc, plan);
 
@@ -711,6 +715,8 @@ struct cff_subset_plan {
       /* global subrs */
       unsigned int dataSize = subset_globalsubrs.total_size ();
       offsets.globalSubrsInfo.offSize = calcOffSize (dataSize);
+      if (unlikely (offsets.globalSubrsInfo.offSize > 4))
+      	return false;
       offsets.globalSubrsInfo.size = CFF1Subrs::calculate_serialized_size (offsets.globalSubrsInfo.offSize, subset_globalsubrs.len, dataSize);
 
       /* local subrs */
@@ -732,6 +738,8 @@ struct cff_subset_plan {
 	  {
 	    offsets.localSubrsInfos[fd].offset = final_size;
 	    offsets.localSubrsInfos[fd].offSize = calcOffSize (dataSize);
+	    if (unlikely (offsets.localSubrsInfos[fd].offSize > 4))
+	      return false;
 	    offsets.localSubrsInfos[fd].size = CFF1Subrs::calculate_serialized_size (offsets.localSubrsInfos[fd].offSize, subset_localsubrs[fd].len, dataSize);
 	  }
 	}
@@ -775,6 +783,8 @@ struct cff_subset_plan {
 	  dictsSize += FontDict::calculate_serialized_size (acc.fontDicts[i], fontSzr);
 
       offsets.FDArrayInfo.offSize = calcOffSize (dictsSize);
+      if (unlikely (offsets.FDArrayInfo.offSize > 4))
+      	return false;
       final_size += CFF1Index::calculate_serialized_size (offsets.FDArrayInfo.offSize, subset_fdcount, dictsSize);
     }
 
@@ -783,6 +793,8 @@ struct cff_subset_plan {
       offsets.charStringsInfo.offset = final_size;
       unsigned int dataSize = subset_charstrings.total_size ();
       offsets.charStringsInfo.offSize = calcOffSize (dataSize);
+      if (unlikely (offsets.charStringsInfo.offSize > 4))
+      	return false;
       final_size += CFF1CharStrings::calculate_serialized_size (offsets.charStringsInfo.offSize, plan->glyphs.len, dataSize);
     }
 

src/hb-subset-cff2.cc

@@ -112,7 +112,11 @@ struct CFF2CSOpSet_Flatten : CFF2CSOpSet<CFF2CSOpSet_Flatten, FlattenParam>
       const BlendArg &arg = env.argStack[i];
       if (arg.blending ())
       {
-	assert ((arg.numValues > 0) && (env.argStack.get_count () >= arg.numValues));
+      	if (unlikely (!((arg.numValues > 0) && (env.argStack.get_count () >= arg.numValues))))
+      	{
+	  env.set_error ();
+	  return;
+	}
 	flatten_blends (arg, i, env, param);
 	i += arg.numValues;
       }
@@ -133,8 +137,12 @@ struct CFF2CSOpSet_Flatten : CFF2CSOpSet<CFF2CSOpSet_Flatten, FlattenParam>
     for (unsigned int j = 0; j < arg.numValues; j++)
     {
       const BlendArg &arg1 = env.argStack[i + j];
-      assert (arg1.blending () && (arg.numValues == arg1.numValues) && (arg1.valueIndex == j) &&
-	      (arg1.deltas.len == env.get_region_count ()));
+      if (unlikely (!((arg1.blending () && (arg.numValues == arg1.numValues) && (arg1.valueIndex == j) &&
+	      (arg1.deltas.len == env.get_region_count ())))))
+      {
+      	env.set_error ();
+      	return;
+      }
       encoder.encode_num (arg1);
     }
     /* flatten deltas for each value */