8071715: Tune font layout engine

Reviewed-by: srl, bae, mschoene

src/share/native/sun/font/layout/ContextualGlyphInsertionProc2.cpp

@@ -107,6 +107,10 @@ le_uint16 ContextualGlyphInsertionProcessor2::processStateEntry(LEGlyphStorage &
 
     le_int16 markIndex = SWAPW(entry->markedInsertionListIndex);
     if (markIndex > 0) {
+        if (markGlyph < 0 || markGlyph >= glyphStorage.getGlyphCount()) {
+           success = LE_INDEX_OUT_OF_BOUNDS_ERROR;
+           return 0;
+        }
         le_int16 count = (flags & cgiMarkedInsertCountMask) >> 5;
         le_bool isKashidaLike = (flags & cgiMarkedIsKashidaLike);
         le_bool isBefore = (flags & cgiMarkInsertBefore);
@@ -115,6 +119,10 @@ le_uint16 ContextualGlyphInsertionProcessor2::processStateEntry(LEGlyphStorage &
 
     le_int16 currIndex = SWAPW(entry->currentInsertionListIndex);
     if (currIndex > 0) {
+        if (currGlyph < 0 || currGlyph >= glyphStorage.getGlyphCount()) {
+           success = LE_INDEX_OUT_OF_BOUNDS_ERROR;
+           return 0;
+        }
         le_int16 count = flags & cgiCurrentInsertCountMask;
         le_bool isKashidaLike = (flags & cgiCurrentIsKashidaLike);
         le_bool isBefore = (flags & cgiCurrentInsertBefore);

src/share/native/sun/font/layout/ContextualGlyphSubstProc.cpp

@@ -76,6 +76,10 @@ ByteOffset ContextualGlyphSubstitutionProcessor::processStateEntry(LEGlyphStorag
   WordOffset currOffset = SWAPW(entry->currOffset);
 
   if (markOffset != 0 && LE_SUCCESS(success)) {
+    if (markGlyph < 0 || markGlyph >= glyphStorage.getGlyphCount()) {
+       success = LE_INDEX_OUT_OF_BOUNDS_ERROR;
+       return 0;
+    }
     LEGlyphID mGlyph = glyphStorage[markGlyph];
     TTGlyphID newGlyph = SWAPW(int16Table.getObject(markOffset + LE_GET_GLYPH(mGlyph), success)); // whew.
 
@@ -83,6 +87,10 @@ ByteOffset ContextualGlyphSubstitutionProcessor::processStateEntry(LEGlyphStorag
   }
 
   if (currOffset != 0) {
+    if (currGlyph < 0 || currGlyph >= glyphStorage.getGlyphCount()) {
+       success = LE_INDEX_OUT_OF_BOUNDS_ERROR;
+       return 0;
+    }
     LEGlyphID thisGlyph = glyphStorage[currGlyph];
     TTGlyphID newGlyph = SWAPW(int16Table.getObject(currOffset + LE_GET_GLYPH(thisGlyph), success)); // whew.
 

src/share/native/sun/font/layout/ContextualGlyphSubstProc2.cpp

@@ -70,17 +70,25 @@ le_uint16 ContextualGlyphSubstitutionProcessor2::processStateEntry(LEGlyphStorag
     if(LE_FAILURE(success)) return 0;
     le_uint16 newState = SWAPW(entry->newStateIndex);
     le_uint16 flags = SWAPW(entry->flags);
-    le_int16 markIndex = SWAPW(entry->markIndex);
-    le_int16 currIndex = SWAPW(entry->currIndex);
+    le_uint16 markIndex = SWAPW(entry->markIndex);
+    le_uint16 currIndex = SWAPW(entry->currIndex);
 
-    if (markIndex != -1) {
+    if (markIndex != 0x0FFFF) {
+        if (markGlyph < 0 || markGlyph >= glyphStorage.getGlyphCount()) {
+           success = LE_INDEX_OUT_OF_BOUNDS_ERROR;
+           return 0;
+        }
         le_uint32 offset = SWAPL(perGlyphTable(markIndex, success));
         LEGlyphID mGlyph = glyphStorage[markGlyph];
         TTGlyphID newGlyph = lookup(offset, mGlyph, success);
         glyphStorage[markGlyph] = LE_SET_GLYPH(mGlyph, newGlyph);
     }
 
-    if (currIndex != -1) {
+    if (currIndex != 0x0FFFF) {
+        if (currGlyph < 0 || currGlyph >= glyphStorage.getGlyphCount()) {
+           success = LE_INDEX_OUT_OF_BOUNDS_ERROR;
+           return 0;
+        }
         le_uint32 offset = SWAPL(perGlyphTable(currIndex, success));
         LEGlyphID thisGlyph = glyphStorage[currGlyph];
         TTGlyphID newGlyph = lookup(offset, thisGlyph, success);

src/share/native/sun/font/layout/IndicRearrangementProcessor.cpp

@@ -70,6 +70,11 @@ ByteOffset IndicRearrangementProcessor::processStateEntry(LEGlyphStorage &glyphS
     ByteOffset newState = SWAPW(entry->newStateOffset);
     IndicRearrangementFlags flags = (IndicRearrangementFlags) SWAPW(entry->flags);
 
+    if (currGlyph < 0 || currGlyph >= glyphStorage.getGlyphCount()) {
+       success = LE_INDEX_OUT_OF_BOUNDS_ERROR;
+       return 0;
+    }
+
     if (flags & irfMarkFirst) {
         firstGlyph = currGlyph;
     }

src/share/native/sun/font/layout/IndicRearrangementProcessor2.cpp

@@ -68,6 +68,11 @@ le_uint16 IndicRearrangementProcessor2::processStateEntry(LEGlyphStorage &glyphS
     le_uint16 newState = SWAPW(entry->newStateIndex); // index to the new state
     IndicRearrangementFlags  flags =  (IndicRearrangementFlags) SWAPW(entry->flags);
 
+    if (currGlyph < 0 || currGlyph >= glyphStorage.getGlyphCount()) {
+       success = LE_INDEX_OUT_OF_BOUNDS_ERROR;
+       return 0;
+    }
+
     if (flags & irfMarkFirst) {
         firstGlyph = currGlyph;
     }

src/share/native/sun/font/layout/LigatureSubstProc.cpp

@@ -73,7 +73,7 @@ ByteOffset LigatureSubstitutionProcessor::processStateEntry(LEGlyphStorage &glyp
   const LigatureSubstitutionStateEntry *entry = entryTable.getAlias(index, success);
 
     ByteOffset newState = SWAPW(entry->newStateOffset);
-    le_int16 flags = SWAPW(entry->flags);
+    le_uint16 flags = SWAPW(entry->flags);
 
     if (flags & lsfSetComponent) {
         if (++m >= nComponents) {

src/share/native/sun/font/layout/StateTableProcessor.cpp

@@ -85,6 +85,7 @@ void StateTableProcessor::process(LEGlyphStorage &glyphStorage, LEErrorCode &suc
         if (currGlyph == glyphCount) {
             // XXX: How do we handle EOT vs. EOL?
             classCode = classCodeEOT;
+            break;
         } else {
             TTGlyphID glyphCode = (TTGlyphID) LE_GET_GLYPH(glyphStorage[currGlyph]);
 

src/share/native/sun/font/layout/StateTableProcessor2.cpp

@@ -103,6 +103,7 @@ void StateTableProcessor2::process(LEGlyphStorage &glyphStorage, LEErrorCode &su
                 if (currGlyph == glyphCount || currGlyph == -1) {
                     // XXX: How do we handle EOT vs. EOL?
                     classCode = classCodeEOT;
+                    break;
                 } else {
                     LEGlyphID gid = glyphStorage[currGlyph];
                     TTGlyphID glyphCode = (TTGlyphID) LE_GET_GLYPH(gid);
@@ -134,6 +135,7 @@ void StateTableProcessor2::process(LEGlyphStorage &glyphStorage, LEErrorCode &su
                 if (currGlyph == glyphCount || currGlyph == -1) {
                     // XXX: How do we handle EOT vs. EOL?
                     classCode = classCodeEOT;
+                    break;
                 } else {
                     LEGlyphID gid = glyphStorage[currGlyph];
                     TTGlyphID glyphCode = (TTGlyphID) LE_GET_GLYPH(gid);
@@ -171,6 +173,7 @@ void StateTableProcessor2::process(LEGlyphStorage &glyphStorage, LEErrorCode &su
                 if (currGlyph == glyphCount || currGlyph == -1) {
                     // XXX: How do we handle EOT vs. EOL?
                     classCode = classCodeEOT;
+                    break;
                 } else if(currGlyph > glyphCount) {
                   // note if > glyphCount, we've run off the end (bad font)
                   currGlyph = glyphCount;
@@ -211,6 +214,7 @@ void StateTableProcessor2::process(LEGlyphStorage &glyphStorage, LEErrorCode &su
                 if (currGlyph == glyphCount || currGlyph == -1) {
                     // XXX: How do we handle EOT vs. EOL?
                     classCode = classCodeEOT;
+                    break;
                 } else {
                     TTGlyphID glyphCode = (TTGlyphID) LE_GET_GLYPH(glyphStorage[currGlyph]);
                     if (glyphCode == 0xFFFF) {

src/share/native/sun/font/layout/StateTables.h

@@ -126,7 +126,7 @@ typedef le_uint8 EntryTableIndex;
 struct StateEntry
 {
     ByteOffset  newStateOffset;
-    le_int16    flags;
+    le_uint16    flags;
 };
 
 typedef le_uint16 EntryTableIndex2;