8028468: Add inlining information into ciReplay

Summary: Allow dump and replay inlining for specified method during a program execution.
Reviewed-by: roland, twisti

hotspot/agent/src/share/classes/sun/jvm/hotspot/ci/ciEnv.java

@@ -95,9 +95,15 @@ public class ciEnv extends VMObject {
     int entryBci = task.osrBci();
     int compLevel = task.compLevel();
     Klass holder = method.getMethodHolder();
-    out.println("compile " + holder.getName().asString() + " " +
-                OopUtilities.escapeString(method.getName().asString()) + " " +
-                method.getSignature().asString() + " " +
-                entryBci + " " + compLevel);
+    out.print("compile " + holder.getName().asString() + " " +
+              OopUtilities.escapeString(method.getName().asString()) + " " +
+              method.getSignature().asString() + " " +
+              entryBci + " " + compLevel);
+    Compile compiler = compilerData();
+    if (compiler != null) {
+      // Dump inlining data.
+      compiler.dumpInlineData(out);
+    }
+    out.println();
   }
 }

hotspot/agent/src/share/classes/sun/jvm/hotspot/opto/Compile.java

@@ -25,6 +25,7 @@
 package sun.jvm.hotspot.opto;
 
 import java.util.*;
+import java.io.PrintStream;
 import sun.jvm.hotspot.ci.*;
 import sun.jvm.hotspot.debugger.*;
 import sun.jvm.hotspot.runtime.*;
@@ -92,4 +93,13 @@ public class Compile extends VMObject {
     }
     return null;
   }
+
+  public void dumpInlineData(PrintStream out) {
+    InlineTree inlTree = ilt();
+    if (inlTree != null) {
+      out.print(" inline " + inlTree.count());
+      inlTree.dumpReplayData(out);
+    }
+  }
+
 }

hotspot/agent/src/share/classes/sun/jvm/hotspot/opto/InlineTree.java

@@ -87,6 +87,11 @@ public class InlineTree extends VMObject {
     return GrowableArray.create(addr, inlineTreeConstructor);
   }
 
+  public int inlineLevel() {
+    JVMState jvms = callerJvms();
+    return (jvms != null) ? jvms.depth() : 0;
+  }
+
   public void printImpl(PrintStream st, int indent) {
     for (int i = 0; i < indent; i++) st.print(" ");
     st.printf(" @ %d ", callerBci());
@@ -101,4 +106,28 @@ public class InlineTree extends VMObject {
   public void print(PrintStream st) {
     printImpl(st, 2);
   }
+
+  // Count number of nodes in this subtree
+  public int count() {
+    int result = 1;
+    GrowableArray<InlineTree> subt = subtrees();
+    for (int i = 0 ; i < subt.length(); i++) {
+      result += subt.at(i).count();
+    }
+    return result;
+  }
+
+  public void dumpReplayData(PrintStream out) {
+    out.printf(" %d %d ", inlineLevel(), callerBci());
+    Method method = (Method)method().getMetadata();
+    Klass holder = method.getMethodHolder();
+    out.print(holder.getName().asString() + " " +
+              OopUtilities.escapeString(method.getName().asString()) + " " +
+              method.getSignature().asString());
+
+    GrowableArray<InlineTree> subt = subtrees();
+    for (int i = 0 ; i < subt.length(); i++) {
+      subt.at(i).dumpReplayData(out);
+    }
+  }
 }

hotspot/agent/src/share/classes/sun/jvm/hotspot/opto/JVMState.java

@@ -88,6 +88,10 @@ public class JVMState extends VMObject {
     return (int)bciField.getValue(getAddress());
   }
 
+  public int depth() {
+    return (int)depthField.getValue(getAddress());
+  }
+
   public JVMState caller() {
     return create(callerField.getValue(getAddress()));
   }

hotspot/src/share/vm/c1/c1_Compilation.hpp

@@ -259,6 +259,9 @@ class Compilation: public StackObj {
   }
 
   ciKlass* cha_exact_type(ciType* type);
+
+  // Dump inlining replay data to the stream.
+  void dump_inline_data(outputStream* out) { /* do nothing now */ }
 };
 
 

hotspot/src/share/vm/ci/ciEnv.cpp

@@ -1147,6 +1147,33 @@ ciInstance* ciEnv::unloaded_ciinstance() {
 
 // Don't change thread state and acquire any locks.
 // Safe to call from VM error reporter.
+
+void ciEnv::dump_compile_data(outputStream* out) {
+  CompileTask* task = this->task();
+  Method* method = task->method();
+  int entry_bci = task->osr_bci();
+  int comp_level = task->comp_level();
+  out->print("compile %s %s %s %d %d",
+                method->klass_name()->as_quoted_ascii(),
+                method->name()->as_quoted_ascii(),
+                method->signature()->as_quoted_ascii(),
+                entry_bci, comp_level);
+  if (compiler_data() != NULL) {
+    if (is_c2_compile(comp_level)) { // C2 or Shark
+#ifdef COMPILER2
+      // Dump C2 inlining data.
+      ((Compile*)compiler_data())->dump_inline_data(out);
+#endif
+    } else if (is_c1_compile(comp_level)) { // C1
+#ifdef COMPILER1
+      // Dump C1 inlining data.
+      ((Compilation*)compiler_data())->dump_inline_data(out);
+#endif
+    }
+  }
+  out->cr();
+}
+
 void ciEnv::dump_replay_data_unsafe(outputStream* out) {
   ResourceMark rm;
 #if INCLUDE_JVMTI
@@ -1160,16 +1187,7 @@ void ciEnv::dump_replay_data_unsafe(outputStream* out) {
   for (int i = 0; i < objects->length(); i++) {
     objects->at(i)->dump_replay_data(out);
   }
-  CompileTask* task = this->task();
-  Method* method = task->method();
-  int entry_bci = task->osr_bci();
-  int comp_level = task->comp_level();
-  // Klass holder = method->method_holder();
-  out->print_cr("compile %s %s %s %d %d",
-                method->klass_name()->as_quoted_ascii(),
-                method->name()->as_quoted_ascii(),
-                method->signature()->as_quoted_ascii(),
-                entry_bci, comp_level);
+  dump_compile_data(out);
   out->flush();
 }
 
@@ -1179,3 +1197,45 @@ void ciEnv::dump_replay_data(outputStream* out) {
     dump_replay_data_unsafe(out);
   )
 }
+
+void ciEnv::dump_replay_data(int compile_id) {
+  static char buffer[O_BUFLEN];
+  int ret = jio_snprintf(buffer, O_BUFLEN, "replay_pid%p_compid%d.log", os::current_process_id(), compile_id);
+  if (ret > 0) {
+    int fd = open(buffer, O_RDWR | O_CREAT | O_TRUNC, 0666);
+    if (fd != -1) {
+      FILE* replay_data_file = os::open(fd, "w");
+      if (replay_data_file != NULL) {
+        fileStream replay_data_stream(replay_data_file, /*need_close=*/true);
+        dump_replay_data(&replay_data_stream);
+        tty->print("# Compiler replay data is saved as: ");
+        tty->print_cr(buffer);
+      } else {
+        tty->print_cr("# Can't open file to dump replay data.");
+      }
+    }
+  }
+}
+
+void ciEnv::dump_inline_data(int compile_id) {
+  static char buffer[O_BUFLEN];
+  int ret = jio_snprintf(buffer, O_BUFLEN, "inline_pid%p_compid%d.log", os::current_process_id(), compile_id);
+  if (ret > 0) {
+    int fd = open(buffer, O_RDWR | O_CREAT | O_TRUNC, 0666);
+    if (fd != -1) {
+      FILE* inline_data_file = os::open(fd, "w");
+      if (inline_data_file != NULL) {
+        fileStream replay_data_stream(inline_data_file, /*need_close=*/true);
+        GUARDED_VM_ENTRY(
+          MutexLocker ml(Compile_lock);
+          dump_compile_data(&replay_data_stream);
+        )
+        replay_data_stream.flush();
+        tty->print("# Compiler inline data is saved as: ");
+        tty->print_cr(buffer);
+      } else {
+        tty->print_cr("# Can't open file to dump inline data.");
+      }
+    }
+  }
+}

hotspot/src/share/vm/ci/ciEnv.hpp

@@ -451,8 +451,11 @@ public:
   void metadata_do(void f(Metadata*)) { _factory->metadata_do(f); }
 
   // Dump the compilation replay data for the ciEnv to the stream.
+  void dump_replay_data(int compile_id);
+  void dump_inline_data(int compile_id);
   void dump_replay_data(outputStream* out);
   void dump_replay_data_unsafe(outputStream* out);
+  void dump_compile_data(outputStream* out);
 };
 
 #endif // SHARE_VM_CI_CIENV_HPP

hotspot/src/share/vm/ci/ciMethod.cpp

@@ -1357,15 +1357,21 @@ ciMethodBlocks  *ciMethod::get_method_blocks() {
 
 #undef FETCH_FLAG_FROM_VM
 
+void ciMethod::dump_name_as_ascii(outputStream* st) {
+  Method* method = get_Method();
+  st->print("%s %s %s",
+            method->klass_name()->as_quoted_ascii(),
+            method->name()->as_quoted_ascii(),
+            method->signature()->as_quoted_ascii());
+}
+
 void ciMethod::dump_replay_data(outputStream* st) {
   ResourceMark rm;
   Method* method = get_Method();
   MethodCounters* mcs = method->method_counters();
-  Klass*  holder = method->method_holder();
-  st->print_cr("ciMethod %s %s %s %d %d %d %d %d",
-               holder->name()->as_quoted_ascii(),
-               method->name()->as_quoted_ascii(),
-               method->signature()->as_quoted_ascii(),
+  st->print("ciMethod ");
+  dump_name_as_ascii(st);
+  st->print_cr(" %d %d %d %d %d",
                mcs == NULL ? 0 : mcs->invocation_counter()->raw_counter(),
                mcs == NULL ? 0 : mcs->backedge_counter()->raw_counter(),
                interpreter_invocation_count(),

hotspot/src/share/vm/ci/ciMethod.hpp

@@ -310,10 +310,13 @@ class ciMethod : public ciMetadata {
   bool is_accessor    () const;
   bool is_initializer () const;
   bool can_be_statically_bound() const           { return _can_be_statically_bound; }
-  void dump_replay_data(outputStream* st);
   bool is_boxing_method() const;
   bool is_unboxing_method() const;
 
+  // Replay data methods
+  void dump_name_as_ascii(outputStream* st);
+  void dump_replay_data(outputStream* st);
+
   // Print the bytecodes of this method.
   void print_codes_on(outputStream* st);
   void print_codes() {

hotspot/src/share/vm/ci/ciReplay.hpp

@@ -29,6 +29,73 @@
 
 // ciReplay
 
+//
+// Replay compilation of a java method by using an information in replay file.
+// Replay inlining decisions during compilation by using an information in inline file.
+//
+// NOTE: these replay functions only exist in debug version of VM.
+//
+// Replay compilation.
+// -------------------
+//
+// Replay data file replay.txt can be created by Serviceability Agent
+// from a core file, see agent/doc/cireplay.html
+//
+// $ java -cp <jdk>/lib/sa-jdi.jar sun.jvm.hotspot.CLHSDB
+// hsdb> attach <jdk>/bin/java ./core
+// hsdb> threads
+// t@10 Service Thread
+// t@9 C2 CompilerThread0
+// t@8 Signal Dispatcher
+// t@7 Finalizer
+// t@6 Reference Handler
+// t@2 main
+// hsdb> dumpreplaydata t@9 > replay.txt
+// hsdb> quit
+//
+// (Note: SA could be also used to extract app.jar and boot.jar files
+//  from core file to replay compilation if only core file is available)
+//
+// Replay data file replay_pid%p.log is also created when VM crashes
+// in Compiler thread during compilation. It is controlled by
+// DumpReplayDataOnError flag which is ON by default.
+//
+// Replay file replay_pid%p_compid%d.log can be created
+// for the specified java method during normal execution using
+// CompileCommand option DumpReplay:
+//
+// -XX:CompileCommand=option,Benchmark::test,DumpReplay
+//
+// In this case the file name has additional compilation id "_compid%d"
+// because the method could be compiled several times.
+//
+// To replay compilation the replay file should be specified:
+//
+// -XX:+ReplayCompiles -XX:ReplayDataFile=replay_pid2133.log
+//
+// VM thread reads data from the file immediately after VM initialization
+// and puts the compilation task on compile queue. After that it goes into
+// wait state (BackgroundCompilation flag is set to false) since there is no
+// a program to execute. VM exits when the compilation is finished.
+//
+//
+// Replay inlining.
+// ----------------
+//
+// Replay inlining file inline_pid%p_compid%d.log is created for
+// a specific java method during normal execution of a java program
+// using CompileCommand option DumpInline:
+//
+// -XX:CompileCommand=option,Benchmark::test,DumpInline
+//
+// To replay inlining the replay file and the method should be specified:
+//
+// -XX:CompileCommand=option,Benchmark::test,ReplayInline -XX:InlineDataFile=inline_pid3244_compid6.log
+//
+// The difference from replay compilation is that replay inlining
+// is performed during normal java program execution.
+//
+
 class ciReplay {
   CI_PACKAGE_ACCESS
 
@@ -37,7 +104,11 @@ class ciReplay {
   static int replay_impl(TRAPS);
 
  public:
+  // Replay specified compilation and exit VM.
   static void replay(TRAPS);
+  // Load inlining decisions from file and use them
+  // during compilation of specified method.
+  static void* load_inline_data(ciMethod* method, int entry_bci, int comp_level);
 
   // These are used by the CI to fill in the cached data from the
   // replay file when replaying compiles.
@@ -48,6 +119,8 @@ class ciReplay {
   static bool is_loaded(Klass* klass);
 
   static bool should_not_inline(ciMethod* method);
+  static bool should_inline(void* data, ciMethod* method, int bci, int inline_depth);
+  static bool should_not_inline(void* data, ciMethod* method, int bci, int inline_depth);
 
 #endif
 };

hotspot/src/share/vm/memory/allocation.cpp

@@ -140,7 +140,7 @@ void ResourceObj::operator delete [](void* p) {
 void ResourceObj::set_allocation_type(address res, allocation_type type) {
     // Set allocation type in the resource object
     uintptr_t allocation = (uintptr_t)res;
-    assert((allocation & allocation_mask) == 0, "address should be aligned to 4 bytes at least");
+    assert((allocation & allocation_mask) == 0, err_msg("address should be aligned to 4 bytes at least: " PTR_FORMAT, res));
     assert(type <= allocation_mask, "incorrect allocation type");
     ResourceObj* resobj = (ResourceObj *)res;
     resobj->_allocation_t[0] = ~(allocation + type);

hotspot/src/share/vm/opto/bytecodeInfo.cpp

@@ -50,7 +50,10 @@ InlineTree::InlineTree(Compile* c,
   _subtrees(c->comp_arena(), 2, 0, NULL),
   _msg(NULL)
 {
-  NOT_PRODUCT(_count_inlines = 0;)
+#ifndef PRODUCT
+  _count_inlines = 0;
+  _forced_inline = false;
+#endif
   if (_caller_jvms != NULL) {
     // Keep a private copy of the caller_jvms:
     _caller_jvms = new (C) JVMState(caller_jvms->method(), caller_tree->caller_jvms());
@@ -81,7 +84,10 @@ InlineTree::InlineTree(Compile* c, ciMethod* callee_method, JVMState* caller_jvm
   _count_inline_bcs(method()->code_size()),
   _msg(NULL)
 {
-  NOT_PRODUCT(_count_inlines = 0;)
+#ifndef PRODUCT
+  _count_inlines = 0;
+  _forced_inline = false;
+#endif
   assert(!UseOldInlining, "do not use for old stuff");
 }
 
@@ -128,9 +134,19 @@ bool InlineTree::should_inline(ciMethod* callee_method, ciMethod* caller_method,
       tty->print_cr("Inlined method is hot: ");
     }
     set_msg("force inline by CompilerOracle");
+    _forced_inline = true;
     return true;
   }
 
+#ifndef PRODUCT
+  int inline_depth = inline_level()+1;
+  if (ciReplay::should_inline(C->replay_inline_data(), callee_method, caller_bci, inline_depth)) {
+    set_msg("force inline by ciReplay");
+    _forced_inline = true;
+    return true;
+  }
+#endif
+
   int size = callee_method->code_size_for_inlining();
 
   // Check for too many throws (and not too huge)
@@ -264,6 +280,18 @@ bool InlineTree::should_not_inline(ciMethod *callee_method,
   }
 
 #ifndef PRODUCT
+  int caller_bci = jvms->bci();
+  int inline_depth = inline_level()+1;
+  if (ciReplay::should_inline(C->replay_inline_data(), callee_method, caller_bci, inline_depth)) {
+    set_msg("force inline by ciReplay");
+    return false;
+  }
+
+  if (ciReplay::should_not_inline(C->replay_inline_data(), callee_method, caller_bci, inline_depth)) {
+    set_msg("disallowed by ciReplay");
+    return true;
+  }
+
   if (ciReplay::should_not_inline(callee_method)) {
     set_msg("disallowed by ciReplay");
     return true;
@@ -343,6 +371,7 @@ bool InlineTree::try_to_inline(ciMethod* callee_method, ciMethod* caller_method,
     }
   }
 
+  _forced_inline = false; // Reset
   if (!should_inline(callee_method, caller_method, caller_bci, profile,
                      wci_result)) {
     return false;
@@ -373,10 +402,10 @@ bool InlineTree::try_to_inline(ciMethod* callee_method, ciMethod* caller_method,
 
     if ((!UseInterpreter || CompileTheWorld) &&
         is_init_with_ea(callee_method, caller_method, C)) {
-
       // Escape Analysis stress testing when running Xcomp or CTW:
       // inline constructors even if they are not reached.
-
+    } else if (forced_inline()) {
+      // Inlining was forced by CompilerOracle or ciReplay
     } else if (profile.count() == 0) {
       // don't inline unreached call sites
        set_msg("call site not reached");
@@ -700,12 +729,28 @@ InlineTree* InlineTree::find_subtree_from_root(InlineTree* root, JVMState* jvms,
   return iltp;
 }
 
+// Count number of nodes in this subtree
+int InlineTree::count() const {
+  int result = 1;
+  for (int i = 0 ; i < _subtrees.length(); i++) {
+    result += _subtrees.at(i)->count();
+  }
+  return result;
+}
+
+void InlineTree::dump_replay_data(outputStream* out) {
+  out->print(" %d %d ", inline_level(), caller_bci());
+  method()->dump_name_as_ascii(out);
+  for (int i = 0 ; i < _subtrees.length(); i++) {
+    _subtrees.at(i)->dump_replay_data(out);
+  }
+}
 
 
 #ifndef PRODUCT
 void InlineTree::print_impl(outputStream* st, int indent) const {
   for (int i = 0; i < indent; i++) st->print(" ");
-  st->print(" @ %d ", caller_bci());
+  st->print(" @ %d", caller_bci());
   method()->print_short_name(st);
   st->cr();
 

hotspot/src/share/vm/opto/compile.cpp

@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "asm/macroAssembler.hpp"
 #include "asm/macroAssembler.inline.hpp"
+#include "ci/ciReplay.hpp"
 #include "classfile/systemDictionary.hpp"
 #include "code/exceptionHandlerTable.hpp"
 #include "code/nmethod.hpp"
@@ -647,6 +648,7 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr
                   _printer(IdealGraphPrinter::printer()),
 #endif
                   _congraph(NULL),
+                  _replay_inline_data(NULL),
                   _late_inlines(comp_arena(), 2, 0, NULL),
                   _string_late_inlines(comp_arena(), 2, 0, NULL),
                   _boxing_late_inlines(comp_arena(), 2, 0, NULL),
@@ -680,6 +682,10 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr
   }
   set_print_assembly(print_opto_assembly);
   set_parsed_irreducible_loop(false);
+
+  if (method()->has_option("ReplayInline")) {
+    _replay_inline_data = ciReplay::load_inline_data(method(), entry_bci(), ci_env->comp_level());
+  }
 #endif
   set_print_inlining(PrintInlining || method()->has_option("PrintInlining") NOT_PRODUCT( || PrintOptoInlining));
   set_print_intrinsics(PrintIntrinsics || method()->has_option("PrintIntrinsics"));
@@ -849,6 +855,15 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr
 #endif
 
   NOT_PRODUCT( verify_barriers(); )
+
+  // Dump compilation data to replay it.
+  if (method()->has_option("DumpReplay")) {
+    env()->dump_replay_data(_compile_id);
+  }
+  if (method()->has_option("DumpInline") && (ilt() != NULL)) {
+    env()->dump_inline_data(_compile_id);
+  }
+
   // Now that we know the size of all the monitors we can add a fixed slot
   // for the original deopt pc.
 
@@ -938,6 +953,7 @@ Compile::Compile( ciEnv* ci_env,
     _dead_node_list(comp_arena()),
     _dead_node_count(0),
     _congraph(NULL),
+    _replay_inline_data(NULL),
     _number_of_mh_late_inlines(0),
     _inlining_progress(false),
     _inlining_incrementally(false),
@@ -3757,6 +3773,16 @@ void Compile::dump_inlining() {
   }
 }
 
+// Dump inlining replay data to the stream.
+// Don't change thread state and acquire any locks.
+void Compile::dump_inline_data(outputStream* out) {
+  InlineTree* inl_tree = ilt();
+  if (inl_tree != NULL) {
+    out->print(" inline %d", inl_tree->count());
+    inl_tree->dump_replay_data(out);
+  }
+}
+
 int Compile::cmp_expensive_nodes(Node* n1, Node* n2) {
   if (n1->Opcode() < n2->Opcode())      return -1;
   else if (n1->Opcode() > n2->Opcode()) return 1;

hotspot/src/share/vm/opto/compile.hpp

@@ -431,6 +431,8 @@ class Compile : public Phase {
   // Are we within a PreserveJVMState block?
   int _preserve_jvm_state;
 
+  void* _replay_inline_data; // Pointer to data loaded from file
+
  public:
 
   outputStream* print_inlining_stream() const {
@@ -465,6 +467,11 @@ class Compile : public Phase {
     print_inlining_stream()->print(ss.as_string());
   }
 
+  void* replay_inline_data() const { return _replay_inline_data; }
+
+  // Dump inlining replay data to the stream.
+  void dump_inline_data(outputStream* out);
+
  private:
   // Matching, CFG layout, allocation, code generation
   PhaseCFG*             _cfg;                   // Results of CFG finding

hotspot/src/share/vm/opto/parse.hpp

@@ -141,6 +141,13 @@ public:
   GrowableArray<InlineTree*> subtrees() { return _subtrees; }
 
   void print_value_on(outputStream* st) const PRODUCT_RETURN;
+
+  bool        _forced_inline;     // Inlining was forced by CompilerOracle or ciReplay
+  bool        forced_inline()     const { return _forced_inline; }
+  // Count number of nodes in this subtree
+  int         count() const;
+  // Dump inlining replay data to the stream.
+  void dump_replay_data(outputStream* out);
 };
 
 

hotspot/src/share/vm/runtime/globals.hpp

@@ -3342,6 +3342,10 @@ class CommandLineFlags {
           "File containing compilation replay information"                  \
           "[default: ./replay_pid%p.log] (%p replaced with pid)")           \
                                                                             \
+   product(ccstr, InlineDataFile, NULL,                                     \
+          "File containing inlining replay information"                     \
+          "[default: ./inline_pid%p.log] (%p replaced with pid)")           \
+                                                                            \
   develop(intx, ReplaySuppressInitializers, 2,                              \
           "Control handling of class initialization during replay: "        \
           "0 - don't do anything special; "                                 \

hotspot/src/share/vm/utilities/vmError.cpp

@@ -1040,7 +1040,7 @@ void VMError::report_and_die() {
     OnError = NULL;
   }
 
-  static bool skip_replay = false;
+  static bool skip_replay = ReplayCompiles; // Do not overwrite file during replay
   if (DumpReplayDataOnError && _thread && _thread->is_Compiler_thread() && !skip_replay) {
     skip_replay = true;
     ciEnv* env = ciEnv::current();