[JENKINS-37874] - Print warnings if Jenkins startup/reload do not reach the COMPLETED state (#2530)

* [JENKINS-37874] - Log SEVERE messages if Jenkins does not reach COMPLETED stage during startup or reload

* [JENKINS-37874] - Add Administrative monitor for the COMPLETED state

* [JENKINS-37874] - Polish log messages

core/src/main/java/jenkins/diagnostics/CompletedInitializationMonitor.java

+/*
+ * The MIT License
+ *
+ * Copyright (c) 2016, CloudBees, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package jenkins.diagnostics;
+
+import hudson.Extension;
+import hudson.init.InitMilestone;
+import hudson.model.AdministrativeMonitor;
+import jenkins.model.Jenkins;
+import org.jenkinsci.Symbol;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+
+/**
+ * Performs monitoring of {@link Jenkins} {@link InitMilestone} status.
+ *
+ * @author Oleg Nenashev
+ * @since TODO
+ */
+@Restricted(NoExternalUse.class)
+@Extension @Symbol("completedInitialization")
+public class CompletedInitializationMonitor extends AdministrativeMonitor {
+    @Override
+    public boolean isActivated() {
+        final Jenkins instance = Jenkins.getInstance();
+        // Safe to check in such way, because monitors are being checked in UI only.
+        // So Jenkins class construction and initialization must be always finished by the call of this extension.
+        return instance.getInitLevel() != InitMilestone.COMPLETED;
+    }
+}

core/src/main/java/jenkins/model/Jenkins.java

@@ -908,6 +908,18 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
                     InitMilestone.ordering()        // forced ordering among key milestones
             );
 
+            // Ensure we reached the final initialization state. Log the error otherwise
+            if (initLevel != InitMilestone.COMPLETED) {
+                LOGGER.log(SEVERE, "Jenkins initialization has not reached the COMPLETED initialization milestone after the startup. " +
+                                "Current state: {0}. " +
+                                "It may cause undefined incorrect behavior in Jenkins plugin relying on this state. " +
+                                "It is likely an issue with the Initialization task graph. " +
+                                "Example: usage of @Initializer(after = InitMilestone.COMPLETED) in a plugin (JENKINS-37759). " +
+                                "Please create a bug in Jenkins bugtracker. ",
+                        initLevel);
+            }
+
+
             if(KILL_AFTER_LOAD)
                 System.exit(0);
 
@@ -3839,6 +3851,18 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
     public void reload() throws IOException, InterruptedException, ReactorException {
         queue.save();
         executeReactor(null, loadTasks());
+
+        // Ensure we reached the final initialization state. Log the error otherwise
+        if (initLevel != InitMilestone.COMPLETED) {
+            LOGGER.log(SEVERE, "Jenkins initialization has not reached the COMPLETED initialization milestone after the configuration reload. " +
+                            "Current state: {0}. " +
+                            "It may cause undefined incorrect behavior in Jenkins plugin relying on this state. " +
+                            "It is likely an issue with the Initialization task graph. " +
+                            "Example: usage of @Initializer(after = InitMilestone.COMPLETED) in a plugin (JENKINS-37759). " +
+                            "Please create a bug in Jenkins bugtracker.",
+                    initLevel);
+        }
+
         User.reload();
         queue.load();
         servletContext.setAttribute("app", this);

core/src/main/resources/jenkins/diagnostics/CompletedInitializationMonitor/message.jelly

+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
+  <div class="warning">
+      <b>${%Warning!}</b>
+      ${%blurb(app.initLevel)}
+      ${%Example: usage of} <code>@Initializer(after = InitMilestone.COMPLETED)</code> ${%in a plugin}
+      (<a href="https://issues.jenkins-ci.org/browse/JENKINS-37759">JENKINS-37759</a>).
+      ${%Please} <a href="https://wiki.jenkins-ci.org/display/JENKINS/How+to+report+an+issue">${%report a bug}</a> ${%in the Jenkins bugtracker}.
+      <!--TODO: Nice2have: cause diagnostics-->
+  </div>
+</j:jelly>

core/src/main/resources/jenkins/diagnostics/CompletedInitializationMonitor/message.properties

+blurb= Jenkins initialization has not reached the COMPLETED initialization milestone after the configuration reload. \
+    Current state is: \"{0}\". \
+    Such invalid state may cause undefined incorrect behavior of Jenkins plugins. \
+    It is likely an issue with the jenkins initialization or reloading task graph.
+
+