[JENKINS-24752] Avoid nodeprovisioner delay after consulting strategies (#4082)

* Avoid nodeprovisioner delay after consulting strategies

* Try another aproach - rate limit the review every second

* Throttle NodeProvisioner#update every second

* Re-indent to limit diff

* Update lastSuggestedReview

* Fix prior merge

* Logging messages

* Supplier

* Ignore additional calls to suggestReviewNow if we have a queued review

core/src/main/java/hudson/slaves/NodeProvisioner.java

@@ -32,6 +32,7 @@ import static hudson.model.LoadStatistics.DECAY;
 import hudson.model.MultiStageTimeSeries.TimeScale;
 import hudson.Extension;
 import jenkins.util.SystemProperties;
+import jenkins.util.Timer;
 import org.jenkinsci.Symbol;
 
 import javax.annotation.Nonnull;
@@ -136,6 +137,7 @@ public class NodeProvisioner {
     private StrategyState provisioningState = null;
 
     private transient volatile long lastSuggestedReview;
+    private transient volatile boolean queuedReview;
 
     /**
      * Exponential moving average of the "planned capacity" over time, which is the number of
@@ -165,19 +167,28 @@ public class NodeProvisioner {
 
     /**
      * Give the {@link NodeProvisioner} a hint that now would be a good time to think about provisioning some nodes.
-     * The hint will be ignored if subjected to excessive pestering by callers.
+     * Hints are throttled to one every second.
      *
      * @since 1.415
      */
     public void suggestReviewNow() {
-        if (System.currentTimeMillis() > lastSuggestedReview + TimeUnit.SECONDS.toMillis(1)) {
-            lastSuggestedReview = System.currentTimeMillis();
-            Computer.threadPoolForRemoting.submit(new Runnable() {
-                public void run() {
+        if (!queuedReview) {
+            long delay = TimeUnit.SECONDS.toMillis(1) - (System.currentTimeMillis() - lastSuggestedReview);
+            if (delay < 0) {
+                lastSuggestedReview = System.currentTimeMillis();
+                Computer.threadPoolForRemoting.submit(() -> {
                     LOGGER.fine(() -> "running suggested review for " + label);
                     update();
-                }
-            });
+                });
+            } else {
+                queuedReview = true;
+                LOGGER.fine(() -> "running suggested review in " + delay + " ms for " + label);
+                Timer.get().schedule(() -> {
+                    lastSuggestedReview = System.currentTimeMillis();
+                    LOGGER.fine(() -> "running suggested review for " + label + " after " + delay + " ms");
+                    update();
+                }, delay, TimeUnit.MILLISECONDS);
+            }
         } else {
             LOGGER.fine(() -> "ignoring suggested review for " + label);
         }
@@ -195,7 +206,7 @@ public class NodeProvisioner {
         provisioningLock.lock();
         try {
             lastSuggestedReview = System.currentTimeMillis();
-
+            queuedReview = false;
             // We need to get the lock on Queue for two reasons:
             // 1. We will potentially adding a lot of nodes and we don't want to fight with Queue#maintain to acquire
             //    the Queue#lock in order to add each node. Much better is to hold the Queue#lock until all nodes
@@ -209,9 +220,7 @@ public class NodeProvisioner {
             // that causes issues in Queue#maintain) we should be able to remove the need for Queue#lock
             //
             // TODO once Nodes#addNode is made lock free, we should be able to remove the requirement for Queue#lock
-            Queue.withLock(new Runnable() {
-                @Override
-                public void run() {
+            Queue.withLock(() -> {
                     Jenkins jenkins = Jenkins.get();
                     // clean up the cancelled launch activity, then count the # of executors that we are about to
                     // bring up.
@@ -311,7 +320,6 @@ public class NodeProvisioner {
                     } else {
                         provisioningState = new StrategyState(snapshot, label, plannedCapacitySnapshot);
                     }
-                }
             });
 
             if (provisioningState != null) {