[HB] Fix possible int overflows during sanitize

src/TODO

@@ -3,4 +3,3 @@
 - cmap14 support in get_glyph callback
 - size_t?
 - Figure out compiler selection (add test for link to libstdc++)
-- Audit sanitize for int overflows

src/hb-open-type-private.hh

@@ -195,6 +195,26 @@ _hb_sanitize_check (SANITIZE_ARG_DEF,
   return ret;
 }
 
+static HB_GNUC_UNUSED inline bool
+_hb_sanitize_array (SANITIZE_ARG_DEF,
+		    const char *base,
+		    unsigned int record_size,
+		    unsigned int len)
+{
+  bool overflows = len >= ((unsigned int) -1) / record_size;
+
+#if HB_DEBUG
+  if (sanitize_depth < HB_DEBUG) \
+    fprintf (stderr, "SANITIZE(%p) %-*d-> array [%p..%p] (%d*%d=%ld bytes) in [%p..%p] -> %s\n", \
+	     base,
+	     sanitize_depth, sanitize_depth,
+	     base, base + (record_size * len), record_size, len, (unsigned long) record_size * len,
+	     context->start, context->end,
+	     !overflows ? "does not overflow" : "OVERFLOWS FAIL");
+#endif
+  return HB_LIKELY (!overflows) && _hb_sanitize_check (SANITIZE_ARG, base, record_size * len);
+}
+
 static HB_GNUC_UNUSED inline bool
 _hb_sanitize_edit (SANITIZE_ARG_DEF,
 		   const char *base HB_GNUC_UNUSED,
@@ -232,6 +252,8 @@ _hb_sanitize_edit (SANITIZE_ARG_DEF,
 /* TODO Optimize this if L is fixed (gcc magic) */
 #define SANITIZE_MEM(B,L) HB_LIKELY (_hb_sanitize_check (SANITIZE_ARG, CONST_CHARP(B), (L)))
 
+#define SANITIZE_ARRAY(A,S,L) HB_LIKELY (_hb_sanitize_array (SANITIZE_ARG, CONST_CHARP(A), S, L))
+
 #define NEUTER(Var, Val) \
 	(SANITIZE_OBJ (Var) && \
 	 _hb_sanitize_edit (SANITIZE_ARG, CONST_CHARP(&(Var)), sizeof (Var)) && \

src/hb-ot-layout-gpos-private.hh

@@ -561,11 +561,12 @@ struct PairPosFormat2
 
   inline bool sanitize (SANITIZE_ARG_DEF) {
     SANITIZE_DEBUG ();
-    return SANITIZE_SELF () && SANITIZE_THIS (coverage) &&
-	   SANITIZE_THIS2 (classDef1, classDef2) &&
-	   SANITIZE_MEM (values,
-			 (valueFormat1.get_size () + valueFormat2.get_size ()) *
-			 class1Count * class2Count); /* XXX overflow (in other places too) */
+    if (!(SANITIZE_SELF () && SANITIZE_THIS (coverage) &&
+	  SANITIZE_THIS2 (classDef1, classDef2))) return false;
+
+    unsigned int record_size =valueFormat1.get_size () + valueFormat2.get_size ();
+    unsigned int len = class1Count * class2Count;
+    return SANITIZE_ARRAY (values, record_size, len);
   }
 
   private:
@@ -883,7 +884,7 @@ struct BaseArray
     SANITIZE_DEBUG ();
     if (!SANITIZE_SELF ()) return false;
     unsigned int count = cols * len;
-    if (!SANITIZE_MEM (matrix, sizeof (matrix[0]) * count)) return false;
+    if (!SANITIZE_ARRAY (matrix, sizeof (matrix[0]), count)) return false;
     for (unsigned int i = 0; i < count; i++)
       if (!SANITIZE_THIS (matrix[i])) return false;
     return true;
@@ -1175,7 +1176,7 @@ struct Mark2Array
     SANITIZE_DEBUG ();
     if (!SANITIZE_SELF ()) return false;
     unsigned int count = cols * len;
-    if (!SANITIZE_MEM (matrix, sizeof (matrix[0]) * count)) return false;
+    if (!SANITIZE_ARRAY (matrix, sizeof (matrix[0]), count)) return false;
     for (unsigned int i = 0; i < count; i++)
       if (!SANITIZE_THIS (matrix[i])) return false;
     return true;