8214059: Undefined behaviour in ADLC

Reviewed-by: shade, kbarrett
Contributed-by: NSimon Tooke <stooke@redhat.com>

src/share/vm/adlc/adlparse.cpp

@@ -2868,7 +2868,8 @@ void ADLParser::ins_encode_parse_block(InstructForm& inst) {
   const char* param = NULL;
   inst._parameters.reset();
   while ((param = inst._parameters.iter()) != NULL) {
-    OperandForm* opForm = (OperandForm*) inst._localNames[param];
+    OpClassForm* opForm = inst._localNames[param]->is_opclass();
+    assert(opForm != NULL, "sanity");
     encoding->add_parameter(opForm->_ident, param);
   }
 
@@ -3338,7 +3339,8 @@ void ADLParser::constant_parse(InstructForm& inst) {
   const char* param = NULL;
   inst._parameters.reset();
   while ((param = inst._parameters.iter()) != NULL) {
-    OperandForm* opForm = (OperandForm*) inst._localNames[param];
+    OpClassForm* opForm = inst._localNames[param]->is_opclass();
+    assert(opForm != NULL, "sanity");
     encoding->add_parameter(opForm->_ident, param);
   }
 

src/share/vm/adlc/dfa.cpp

@@ -757,19 +757,27 @@ const char *Expr::compute_expr(const Expr *c1, const Expr *c2) {
 }
 
 int Expr::compute_min(const Expr *c1, const Expr *c2) {
-  int result = c1->_min_value + c2->_min_value;
-  assert( result >= 0, "Invalid cost computation");
+  int v1 = c1->_min_value;
+  int v2 = c2->_min_value;
+  assert(0 <= v2 && v2 <= Expr::Max, "sanity");
+  assert(v1 <= Expr::Max - v2, "Invalid cost computation");
 
-  return result;
+  return v1 + v2;
 }
 
+
 int Expr::compute_max(const Expr *c1, const Expr *c2) {
-  int result = c1->_max_value + c2->_max_value;
-  if( result < 0 ) {  // check for overflow
-    result = Expr::Max;
+  int v1 = c1->_max_value;
+  int v2 = c2->_max_value;
+
+  // Check for overflow without producing UB. If v2 is positive
+  // and not larger than Max, the subtraction cannot underflow.
+  assert(0 <= v2 && v2 <= Expr::Max, "sanity");
+  if (v1 > Expr::Max - v2) {
+    return Expr::Max;
   }
 
-  return result;
+  return v1 + v2;
 }
 
 void Expr::print() const {

src/share/vm/adlc/formssel.cpp

@@ -921,7 +921,8 @@ void InstructForm::build_components() {
   const char *name;
   const char *kill_name = NULL;
   for (_parameters.reset(); (name = _parameters.iter()) != NULL;) {
-    OperandForm *opForm = (OperandForm*)_localNames[name];
+    OpClassForm *opForm = _localNames[name]->is_opclass();
+    assert(opForm != NULL, "sanity");
 
     Effect* e = NULL;
     {
@@ -938,7 +939,8 @@ void InstructForm::build_components() {
       // complex so simply enforce the restriction during parse.
       if (kill_name != NULL &&
           e->isa(Component::TEMP) && !e->isa(Component::DEF)) {
-        OperandForm* kill = (OperandForm*)_localNames[kill_name];
+        OpClassForm* kill = _localNames[kill_name]->is_opclass();
+        assert(kill != NULL, "sanity");
         globalAD->syntax_err(_linenum, "%s: %s %s must be at the end of the argument list\n",
                              _ident, kill->_ident, kill_name);
       } else if (e->isa(Component::KILL) && !e->isa(Component::USE)) {
@@ -2339,7 +2341,8 @@ void OperandForm::build_components() {
   // Add parameters that "do not appear in match rule".
   const char *name;
   for (_parameters.reset(); (name = _parameters.iter()) != NULL;) {
-    OperandForm *opForm = (OperandForm*)_localNames[name];
+    OpClassForm *opForm = _localNames[name]->is_opclass();
+    assert(opForm != NULL, "sanity");
 
     if ( _components.operand_position(name) == -1 ) {
       _components.insert(name, opForm->_ident, Component::INVALID, false);