[ubsan] Use unsigned int instead enum where needed (#1270)

Actually the check is right,

On -myanmar.hh, on that particular switch, OT_C is indic_category_t
but OT_D is myanmar_category_t so we are mixing the types in one variable.

And on -arabic.cc, step can goes one number higher than step_t enum in the
loop so we are actually using it as an unsinged int.

.circleci/config.yml

@@ -168,7 +168,7 @@ jobs:
       - run: apt update || true
       - run: apt install -y clang lld binutils libtool autoconf automake make pkg-config ragel libfreetype6-dev libglib2.0-dev libcairo2-dev libicu-dev libgraphite2-dev python python-pip
       - run: pip install fonttools
-      - run: CPPFLAGS="-fsanitize=undefined -fno-sanitize=enum" LDFLAGS="-fsanitize=undefined -fno-sanitize=enum -O1 -g -fno-omit-frame-pointer" CFLAGS="-fsanitize=undefined -fno-sanitize=enum -O1 -g -fno-omit-frame-pointer" CXXFLAGS="-fsanitize=undefined -O1 -g -fno-omit-frame-pointer" LD=ld.lld CC=clang CXX=clang++ ./autogen.sh --with-freetype --with-glib --with-cairo --with-icu --with-graphite2
+      - run: CPPFLAGS="-fsanitize=undefined" LDFLAGS="-fsanitize=undefined -O1 -g -fno-omit-frame-pointer" CFLAGS="-fsanitize=undefined -O1 -g -fno-omit-frame-pointer" CXXFLAGS="-fsanitize=undefined -O1 -g -fno-omit-frame-pointer" LD=ld.lld CC=clang CXX=clang++ ./autogen.sh --with-freetype --with-glib --with-cairo --with-icu --with-graphite2
       - run: make -j32
       - run: make check || .ci/fail.sh | asan_symbolize | c++filt
 

src/hb-ot-shape-complex-arabic.cc

@@ -458,9 +458,9 @@ apply_stch (const hb_ot_shape_plan_t *plan,
 
   int sign = font->x_scale < 0 ? -1 : +1;
   unsigned int extra_glyphs_needed = 0; // Set during MEASURE, used during CUT
-  typedef enum { MEASURE, CUT } step_t;
+  enum { MEASURE, CUT } /* step_t */;
 
-  for (step_t step = MEASURE; step <= CUT; step = (step_t) (step + 1))
+  for (unsigned int step = MEASURE; step <= CUT; step = step + 1)
   {
     unsigned int count = buffer->len;
     hb_glyph_info_t *info = buffer->info;

src/hb-ot-shape-complex-myanmar.hh

@@ -64,42 +64,42 @@ set_myanmar_properties (hb_glyph_info_t &info)
 {
   hb_codepoint_t u = info.codepoint;
   unsigned int type = hb_indic_get_categories (u);
-  indic_category_t cat = (indic_category_t) (type & 0x7Fu);
+  unsigned int cat = type & 0x7Fu;
   indic_position_t pos = (indic_position_t) (type >> 8);
 
   /* Myanmar
    * https://docs.microsoft.com/en-us/typography/script-development/myanmar#analyze
    */
   if (unlikely (hb_in_range<hb_codepoint_t> (u, 0xFE00u, 0xFE0Fu)))
-    cat = (indic_category_t) OT_VS;
+    cat = OT_VS;
 
   switch (u)
   {
     case 0x104Eu:
-      cat = (indic_category_t) OT_C; /* The spec says C, IndicSyllableCategory doesn't have. */
+      cat = OT_C; /* The spec says C, IndicSyllableCategory doesn't have. */
       break;
 
     case 0x002Du: case 0x00A0u: case 0x00D7u: case 0x2012u:
     case 0x2013u: case 0x2014u: case 0x2015u: case 0x2022u:
     case 0x25CCu: case 0x25FBu: case 0x25FCu: case 0x25FDu:
     case 0x25FEu:
-      cat = (indic_category_t) OT_GB;
+      cat = OT_GB;
       break;
 
     case 0x1004u: case 0x101Bu: case 0x105Au:
-      cat = (indic_category_t) OT_Ra;
+      cat = OT_Ra;
       break;
 
     case 0x1032u: case 0x1036u:
-      cat = (indic_category_t) OT_A;
+      cat = OT_A;
       break;
 
     case 0x1039u:
-      cat = (indic_category_t) OT_H;
+      cat = OT_H;
       break;
 
     case 0x103Au:
-      cat = (indic_category_t) OT_As;
+      cat = OT_As;
       break;
 
     case 0x1041u: case 0x1042u: case 0x1043u: case 0x1044u:
@@ -107,47 +107,47 @@ set_myanmar_properties (hb_glyph_info_t &info)
     case 0x1049u: case 0x1090u: case 0x1091u: case 0x1092u:
     case 0x1093u: case 0x1094u: case 0x1095u: case 0x1096u:
     case 0x1097u: case 0x1098u: case 0x1099u:
-      cat = (indic_category_t) OT_D;
+      cat = OT_D;
       break;
 
     case 0x1040u:
-      cat = (indic_category_t) OT_D; /* XXX The spec says D0, but Uniscribe doesn't seem to do. */
+      cat = OT_D; /* XXX The spec says D0, but Uniscribe doesn't seem to do. */
       break;
 
     case 0x103Eu: case 0x1060u:
-      cat = (indic_category_t) OT_MH;
+      cat = OT_MH;
       break;
 
     case 0x103Cu:
-      cat = (indic_category_t) OT_MR;
+      cat = OT_MR;
       break;
 
     case 0x103Du: case 0x1082u:
-      cat = (indic_category_t) OT_MW;
+      cat = OT_MW;
       break;
 
     case 0x103Bu: case 0x105Eu: case 0x105Fu:
-      cat = (indic_category_t) OT_MY;
+      cat = OT_MY;
       break;
 
     case 0x1063u: case 0x1064u: case 0x1069u: case 0x106Au:
     case 0x106Bu: case 0x106Cu: case 0x106Du: case 0xAA7Bu:
-      cat = (indic_category_t) OT_PT;
+      cat = OT_PT;
       break;
 
     case 0x1038u: case 0x1087u: case 0x1088u: case 0x1089u:
     case 0x108Au: case 0x108Bu: case 0x108Cu: case 0x108Du:
     case 0x108Fu: case 0x109Au: case 0x109Bu: case 0x109Cu:
-      cat = (indic_category_t) OT_SM;
+      cat = OT_SM;
       break;
 
     case 0x104Au: case 0x104Bu:
-      cat = (indic_category_t) OT_P;
+      cat = OT_P;
       break;
 
     case 0xAA74u: case 0xAA75u: case 0xAA76u:
       /* https://github.com/roozbehp/unicode-data/issues/3 */
-      cat = (indic_category_t) OT_C;
+      cat = OT_C;
       break;
   }
 
@@ -155,15 +155,15 @@ set_myanmar_properties (hb_glyph_info_t &info)
   {
     switch ((int) pos)
     {
-      case POS_PRE_C:	cat = (indic_category_t) OT_VPre;
-			pos = POS_PRE_M;                  break;
-      case POS_ABOVE_C:	cat = (indic_category_t) OT_VAbv; break;
-      case POS_BELOW_C:	cat = (indic_category_t) OT_VBlw; break;
-      case POS_POST_C:	cat = (indic_category_t) OT_VPst; break;
+      case POS_PRE_C:	cat = OT_VPre;
+			pos = POS_PRE_M; break;
+      case POS_ABOVE_C:	cat = OT_VAbv;   break;
+      case POS_BELOW_C:	cat = OT_VBlw;   break;
+      case POS_POST_C:	cat = OT_VPst;   break;
     }
   }
 
-  info.myanmar_category() = (myanmar_category_t) cat;
+  info.myanmar_category() = cat;
   info.myanmar_position() = pos;
 }