8222670: pathological case of JIT recompilation and code cache bloat

Summary: Prevent downgraded compilation tasks from recompiling.
Reviewed-by: sgehwolf, thartmann, andrew

src/share/vm/compiler/compileBroker.cpp

 /*
- * Copyright (c) 1999, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -751,8 +751,10 @@ CompileTask* CompileQueue::get() {
     No_Safepoint_Verifier nsv;
     task = CompilationPolicy::policy()->select_task(this);
   }
+  if (task != NULL) {
     remove(task);
     purge_stale_tasks(); // may temporarily release MCQ lock
+  }
   return task;
 }
 

src/share/vm/compiler/compileBroker.hpp

 /*
- * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -339,7 +339,6 @@ class CompileBroker: AllStatic {
 
   static CompilerThread* make_compiler_thread(const char* name, CompileQueue* queue, CompilerCounters* counters, AbstractCompiler* comp, TRAPS);
   static void init_compiler_threads(int c1_compiler_count, int c2_compiler_count);
-  static bool compilation_is_complete  (methodHandle method, int osr_bci, int comp_level);
   static bool compilation_is_prohibited(methodHandle method, int osr_bci, int comp_level);
   static bool is_compile_blocking      ();
   static void preload_classes          (methodHandle method, TRAPS);
@@ -389,6 +388,7 @@ class CompileBroker: AllStatic {
     return NULL;
   }
 
+  static bool compilation_is_complete(methodHandle method, int osr_bci, int comp_level);
   static bool compilation_is_in_queue(methodHandle method);
   static int queue_size(int comp_level) {
     CompileQueue *q = compile_queue(comp_level);

src/share/vm/oops/instanceKlass.cpp

 /*
- * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2925,6 +2925,13 @@ void InstanceKlass::adjust_default_methods(InstanceKlass* holder, bool* trace_na
 
 // On-stack replacement stuff
 void InstanceKlass::add_osr_nmethod(nmethod* n) {
+#ifndef PRODUCT
+  if (TieredCompilation) {
+      nmethod * prev = lookup_osr_nmethod(n->method(), n->osr_entry_bci(), n->comp_level(), true);
+      assert(prev == NULL || !prev->is_in_use(),
+      "redundunt OSR recompilation detected. memory leak in CodeCache!");
+  }
+#endif
   // only one compilation can be active
   NEEDS_CLEANUP
   // This is a short non-blocking critical region, so the no safepoint check is ok.
@@ -3046,7 +3053,9 @@ nmethod* InstanceKlass::lookup_osr_nmethod(const Method* m, int bci, int comp_le
     osr = osr->osr_link();
   }
   OsrList_lock->unlock();
-  if (best != NULL && best->comp_level() >= comp_level && match_level == false) {
+
+  assert(match_level == false || best == NULL, "shouldn't pick up anything if match_level is set");
+  if (best != NULL && best->comp_level() >= comp_level) {
     return best;
   }
   return NULL;

src/share/vm/prims/whitebox.cpp

 /*
- * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -633,6 +633,25 @@ WB_ENTRY(void, WB_ClearMethodState(JNIEnv* env, jobject o, jobject method))
   }
 WB_END
 
+WB_ENTRY(void, WB_MarkMethodProfiled(JNIEnv* env, jobject o, jobject method))
+  jmethodID jmid = reflected_method_to_jmid(thread, env, method);
+  CHECK_JNI_EXCEPTION(env);
+  methodHandle mh(THREAD, Method::checked_resolve_jmethod_id(jmid));
+
+  MethodData* mdo = mh->method_data();
+  if (mdo == NULL) {
+    Method::build_interpreter_method_data(mh, CHECK_AND_CLEAR);
+    mdo = mh->method_data();
+  }
+  mdo->init();
+  InvocationCounter* icnt = mdo->invocation_counter();
+  InvocationCounter* bcnt = mdo->backedge_counter();
+  // set i-counter according to AdvancedThresholdPolicy::is_method_profiled
+  // because SimpleThresholdPolicy::call_predicate_helper uses > in jdk8u, that's why we need to plus one.
+  icnt->set(InvocationCounter::wait_for_compile, Tier4MinInvocationThreshold + 1);
+  bcnt->set(InvocationCounter::wait_for_compile, Tier4CompileThreshold + 1);
+WB_END
+
 template <typename T>
 static bool GetVMFlag(JavaThread* thread, JNIEnv* env, jstring name, T* value, bool (*TAt)(const char*, T*)) {
   if (name == NULL) {
@@ -1123,6 +1142,8 @@ static JNINativeMethod methods[] = {
       CC"(Ljava/lang/reflect/Executable;II)Z",        (void*)&WB_EnqueueMethodForCompilation},
   {CC"clearMethodState",
       CC"(Ljava/lang/reflect/Executable;)V",          (void*)&WB_ClearMethodState},
+  {CC"markMethodProfiled",
+      CC"(Ljava/lang/reflect/Executable;)V",          (void*)&WB_MarkMethodProfiled},
   {CC"setBooleanVMFlag",   CC"(Ljava/lang/String;Z)V",(void*)&WB_SetBooleanVMFlag},
   {CC"setIntxVMFlag",      CC"(Ljava/lang/String;J)V",(void*)&WB_SetIntxVMFlag},
   {CC"setUintxVMFlag",     CC"(Ljava/lang/String;J)V",(void*)&WB_SetUintxVMFlag},

src/share/vm/runtime/advancedThresholdPolicy.cpp

 /*
- * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -196,6 +196,16 @@ CompileTask* AdvancedThresholdPolicy::select_task(CompileQueue* compile_queue) {
 
   if (max_task->comp_level() == CompLevel_full_profile && TieredStopAtLevel > CompLevel_full_profile
       && is_method_profiled(max_method)) {
+
+    if (CompileBroker::compilation_is_complete(max_method, max_task->osr_bci(), CompLevel_limited_profile)) {
+      if (PrintTieredEvents) {
+        print_event(REMOVE_FROM_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
+      }
+      compile_queue->remove_and_mark_stale(max_task);
+      max_method->clear_queued_for_compilation();
+      return NULL;
+    }
+
     max_task->set_comp_level(CompLevel_limited_profile);
     if (PrintTieredEvents) {
       print_event(UPDATE_IN_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());

test/compiler/tiered/Level2RecompilationTest.java

+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test Level2RecompilationTest
+ * @summary Test downgrading mechanism from level 3 to level 2 for those profiled methods.
+ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox
+ * @build Level2RecompilationTest
+ * @run main ClassFileInstaller sun.hotspot.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+TieredCompilation
+ *                   -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:-UseCounterDecay
+ *                   -XX:CompileCommand=compileonly,SimpleTestCase$Helper::*
+ *                   -XX:CompileCommand=print,SimpleTestCase$Helper::*
+ *                   Level2RecompilationTest
+ */
+public class Level2RecompilationTest extends CompLevelsTest {
+    public static void main(String[] args) throws Throwable {
+        if (CompilerWhiteBoxTest.skipOnTieredCompilation(false)) {
+            throw new RuntimeException("Test isn't applicable for non-tiered mode");
+        }
+        String[] testcases = {"METHOD_TEST", "OSR_STATIC_TEST"};
+        CompilerWhiteBoxTest.main(Level2RecompilationTest::new, testcases);
+    }
+
+    protected Level2RecompilationTest(TestCase testCase) {
+        super(testCase);
+        // to prevent inlining of #method
+        WHITE_BOX.testSetDontInlineMethod(method, true);
+    }
+
+    @Override
+    protected void test() throws Exception {
+        if (skipXcompOSR()) {
+          return;
+        }
+
+        checkNotCompiled();
+        int bci = WHITE_BOX.getMethodEntryBci(method);
+        WHITE_BOX.markMethodProfiled(method);
+        if (testCase.isOsr()) {
+            // for OSR compilation, it must be the begin of a BB.
+            // c1_GraphBulider.cpp:153  assert(method()->bci_block_start().at(cur_bci), ...
+            bci = 0;
+        }
+
+        WHITE_BOX.enqueueMethodForCompilation(method, COMP_LEVEL_FULL_PROFILE, bci);
+        checkCompiled();
+        checkLevel(COMP_LEVEL_LIMITED_PROFILE, getCompLevel());
+
+        for (int i=0; i<100; ++i) {
+            WHITE_BOX.enqueueMethodForCompilation(method, COMP_LEVEL_FULL_PROFILE, bci);
+            waitBackgroundCompilation();
+            checkLevel(COMP_LEVEL_LIMITED_PROFILE, getCompLevel());
+        }
+    }
+
+    @Override
+    protected void checkLevel(int expected, int actual) {
+        if (expected == COMP_LEVEL_FULL_PROFILE
+                && actual == COMP_LEVEL_LIMITED_PROFILE) {
+            // for simple method full_profile may be replaced by limited_profile
+            if (IS_VERBOSE) {
+                System.out.printf("Level check: full profiling was replaced "
+                        + "by limited profiling. Expected: %d, actual:%d\n",
+                        expected, actual);
+            }
+            return;
+        }
+        super.checkLevel(expected, actual);
+    }
+}
+

test/compiler/whitebox/CompilerWhiteBoxTest.java

 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -328,11 +328,11 @@ public abstract class CompilerWhiteBoxTest {
             return;
         }
         final Object obj = new Object();
-        for (int i = 0; i < 10
+        for (int i = 0; i < 100
                 && WHITE_BOX.isMethodQueuedForCompilation(executable); ++i) {
             synchronized (obj) {
                 try {
-                    obj.wait(1000);
+                    obj.wait(100);
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
                 }

test/testlibrary/whitebox/sun/hotspot/WhiteBox.java

@@ -162,6 +162,7 @@ public class WhiteBox {
   }
   public native boolean enqueueMethodForCompilation(Executable method, int compLevel, int entry_bci);
   public native void    clearMethodState(Executable method);
+  public native void    markMethodProfiled(Executable method);
   public native int     getMethodEntryBci(Executable method);
   public native Object[] getNMethod(Executable method, boolean isOsr);