4766230: Hotspot vtable inconsistencies cause core dumps. 6579515. 6582242.

Reviewed-by: kamg, coleenp

src/share/vm/classfile/classFileParser.cpp

@@ -2747,9 +2747,10 @@ instanceKlassHandle ClassFileParser::parseClassFile(symbolHandle name,
                                                       super_klass(),
                                                       methods(),
                                                       access_flags,
-                                                      class_loader(),
-                                                      class_name(),
-                                                      local_interfaces());
+                                                      class_loader,
+                                                      class_name,
+                                                      local_interfaces(),
+                                                      CHECK_(nullHandle));
 
     // Size of Java itable (in words)
     itable_size = access_flags.is_interface() ? 0 : klassItable::compute_itable_size(transitive_interfaces);

src/share/vm/oops/instanceKlass.cpp

@@ -1859,6 +1859,25 @@ bool instanceKlass::is_same_class_package(oop class_loader1, symbolOop class_nam
   }
 }
 
+// Returns true iff super_method can be overridden by a method in targetclassname
+// See JSL 3rd edition 8.4.6.1
+// Assumes name-signature match
+// "this" is instanceKlass of super_method which must exist
+// note that the instanceKlass of the method in the targetclassname has not always been created yet
+bool instanceKlass::is_override(methodHandle super_method, Handle targetclassloader, symbolHandle targetclassname, TRAPS) {
+   // Private methods can not be overridden
+   if (super_method->is_private()) {
+     return false;
+   }
+   // If super method is accessible, then override
+   if ((super_method->is_protected()) ||
+       (super_method->is_public())) {
+     return true;
+   }
+   // Package-private methods are not inherited outside of package
+   assert(super_method->is_package_private(), "must be package private");
+   return(is_same_class_package(targetclassloader(), targetclassname()));
+}
 
 jint instanceKlass::compute_modifier_flags(TRAPS) const {
   klassOop k = as_klassOop();

src/share/vm/oops/instanceKlass.hpp

@@ -303,6 +303,9 @@ class instanceKlass: public Klass {
     inner_class_next_offset = 4
   };
 
+  // method override check
+  bool is_override(methodHandle super_method, Handle targetclassloader, symbolHandle targetclassname, TRAPS);
+
   // package
   bool is_same_class_package(klassOop class2);
   bool is_same_class_package(oop classloader2, symbolOop classname2);

src/share/vm/oops/klassVtable.hpp

@@ -70,8 +70,9 @@ class klassVtable : public ResourceObj {
   // conputes vtable length (in words) and the number of miranda methods
   static void compute_vtable_size_and_num_mirandas(int &vtable_length, int &num_miranda_methods,
                                                    klassOop super, objArrayOop methods,
-                                                   AccessFlags class_flags, oop classloader,
-                                                   symbolOop classname, objArrayOop local_interfaces);
+                                                   AccessFlags class_flags, Handle classloader,
+                                                   symbolHandle classname, objArrayOop local_interfaces,
+                                                   TRAPS);
 
   // RedefineClasses() API support:
   // If any entry of this vtable points to any of old_methods,
@@ -111,14 +112,16 @@ class klassVtable : public ResourceObj {
  protected:
   friend class vtableEntry;
  private:
+  enum { VTABLE_TRANSITIVE_OVERRIDE_VERSION = 51 } ;
   void copy_vtable_to(vtableEntry* start);
   int  initialize_from_super(KlassHandle super);
   int  index_of(methodOop m, int len) const; // same as index_of, but search only up to len
   void put_method_at(methodOop m, int index);
-  static bool needs_new_vtable_entry(methodOop m, klassOop super, oop classloader, symbolOop classname, AccessFlags access_flags);
-  AccessType vtable_accessibility_at(int i);
+  static bool needs_new_vtable_entry(methodHandle m, klassOop super, Handle classloader, symbolHandle classname, AccessFlags access_flags, TRAPS);
 
-  bool update_super_vtable(instanceKlass* klass, methodHandle target_method, int super_vtable_len, bool checkconstraints, TRAPS);
+  bool update_inherited_vtable(instanceKlass* klass, methodHandle target_method, int super_vtable_len, bool checkconstraints, TRAPS);
+ instanceKlass* find_transitive_override(instanceKlass* initialsuper, methodHandle target_method, int vtable_index,
+                                         Handle target_loader, symbolHandle target_classname, Thread* THREAD);
 
   // support for miranda methods
   bool is_miranda_entry_at(int i);