8223505: Better pattern compilation

Reviewed-by: rriggs, lancea, dfuchs, mschoene

src/java.xml/share/classes/com/sun/org/apache/xpath/internal/compiler/XPathParser.java

 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  */
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -34,6 +34,7 @@ import com.sun.org.apache.xpath.internal.res.XPATHErrorResources;
  * Tokenizes and parses XPath expressions. This should really be named
  * XPathParserImpl, and may be renamed in the future.
  * @xsl.usage general
+ * @LastModified: May 2019
  */
 public class XPathParser
 {
@@ -71,6 +72,9 @@ public class XPathParser
   protected final static int FILTER_MATCH_PRIMARY    = 1;
   protected final static int FILTER_MATCH_PREDICATES = 2;
 
+  // counts open predicates
+  private int countPredicate;
+
   /**
    * The parser constructor.
    */
@@ -157,6 +161,9 @@ public class XPathParser
           }
           else
                 throw e;
+    } catch (StackOverflowError sof) {
+        error(XPATHErrorResources.ER_PREDICATE_TOO_MANY_OPEN,
+              new Object[]{m_token, m_queueMark, countPredicate});
     }
 
     compiler.shrink();
@@ -190,7 +197,12 @@ public class XPathParser
     m_ops.setOp(OpMap.MAPINDEX_LENGTH, 2);
 
     nextToken();
-    Pattern();
+    try {
+        Pattern();
+    } catch (StackOverflowError sof) {
+        error(XPATHErrorResources.ER_PREDICATE_TOO_MANY_OPEN,
+              new Object[]{m_token, m_queueMark, countPredicate});
+    }
 
     if (null != m_token)
     {
@@ -741,7 +753,7 @@ public class XPathParser
    */
   protected void Expr() throws javax.xml.transform.TransformerException
   {
-    OrExpr();
+       OrExpr();
   }
 
   /**
@@ -1883,11 +1895,12 @@ public class XPathParser
    */
   protected void Predicate() throws javax.xml.transform.TransformerException
   {
-
     if (tokenIs('['))
     {
+      countPredicate++;
       nextToken();
       PredicateExpr();
+      countPredicate--;
       consumeExpected(']');
     }
   }

src/java.xml/share/classes/com/sun/org/apache/xpath/internal/res/XPATHErrorResources.java

 /*
- * reserved comment block
- * DO NOT REMOVE OR ALTER!
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
  */
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -32,6 +31,7 @@ import java.util.ListResourceBundle;
   * Also you need to  update the count of messages(MAX_CODE)or
  * the count of warnings(MAX_WARNING) [ Information purpose only]
  * @xsl.usage advanced
+ * @LastModified: May 2019
  */
 public class XPATHErrorResources extends ListResourceBundle
 {
@@ -150,6 +150,8 @@ public class XPATHErrorResources extends ListResourceBundle
          "ER_FOUND_COMMA_BUT_NO_FOLLOWING_ARG";
   public static final String ER_PREDICATE_ILLEGAL_SYNTAX =
          "ER_PREDICATE_ILLEGAL_SYNTAX";
+  public static final String ER_PREDICATE_TOO_MANY_OPEN =
+         "ER_PREDICATE_TOO_MANY_OPEN";
   public static final String ER_ILLEGAL_AXIS_NAME = "ER_ILLEGAL_AXIS_NAME";
   public static final String ER_UNKNOWN_NODETYPE = "ER_UNKNOWN_NODETYPE";
   public static final String ER_PATTERN_LITERAL_NEEDS_BE_QUOTED =
@@ -464,6 +466,9 @@ public static final String ER_IGNORABLE_WHITESPACE_NOT_HANDLED =
   { ER_PREDICATE_ILLEGAL_SYNTAX,
       "'..[predicate]' or '.[predicate]' is illegal syntax.  Use 'self::node()[predicate]' instead."},
 
+  { ER_PREDICATE_TOO_MANY_OPEN,
+      "Stack overflow while parsing {0} at {1}. Too many open predicates {2}."},
+
   { ER_ILLEGAL_AXIS_NAME,
      "illegal axis name: {0}"},