[JENKINS-57528] - Load detached plugins which are implicit deps of other...

[JENKINS-57528] - Load detached plugins which are implicit deps of other plugins at startup even w/o UC (#4000)

* Do not warn repeatedly of the same missing update site dependency.
This happens when locally testing split plugins (despite their presence in WEB-INF/detached-plugins/*.hpi).

(cherry picked from commit 918a0fd2211210a158279b2f19690c86801fe4a1)

* Detached plugins must be kept up to date with security advisories.

(cherry picked from commit e08d28a69897ff94811e02040255a9d266c44d0d)

* Load detached plugins which are implicit dependencies of other plugins at startup, even if the update center was not used.

(cherry picked from commit ad7a36001d30a6174b25e616fc3c215dd24919d1)

* Adjusted expected version after e08d28a69897ff94811e02040255a9d266c44d0d.

(cherry picked from commit 84f939cebd1a0d03633e7b7c01e71ac387fb6463)

* e08d28a69897ff94811e02040255a9d266c44d0d exposed tests which had been incorrectly storing plugins as *.hpi rather than *.jpi.

(cherry picked from commit a0abad7ec474b28d2d388f075bc54f961046b09a)

* Test failure after ad7a36001d30a6174b25e616fc3c215dd24919d1.

(cherry picked from commit 0084e8dbdddf93bb1353516c1fe20d807577631f)

* Another test failure after ad7a36001d30a6174b25e616fc3c215dd24919d1; fewer installDetachedPlugin calls are now required.

(cherry picked from commit b65bb54e9975a7c77103bf7cf4da82295498f308)

core/src/main/java/hudson/ClassicPluginStrategy.java

@@ -251,14 +251,18 @@ public class ClassicPluginStrategy implements PluginStrategy {
                 createClassLoader(paths, dependencyLoader, atts), disableFile, dependencies, optionalDependencies);
     }
 
-    private static void fix(Attributes atts, List<PluginWrapper.Dependency> optionalDependencies) {
+    private void fix(Attributes atts, List<PluginWrapper.Dependency> optionalDependencies) {
         String pluginName = atts.getValue("Short-Name");
         
         String jenkinsVersion = atts.getValue("Jenkins-Version");
         if (jenkinsVersion==null)
             jenkinsVersion = atts.getValue("Hudson-Version");
-        
-        optionalDependencies.addAll(DetachedPluginsUtil.getImpliedDependencies(pluginName, jenkinsVersion));
+
+        for (Dependency d : DetachedPluginsUtil.getImpliedDependencies(pluginName, jenkinsVersion)) {
+            LOGGER.fine(() -> "implied dep " + pluginName + " → " + d.shortName);
+            pluginManager.considerDetachedPlugin(d.shortName);
+            optionalDependencies.add(d);
+        }
     }
 
     /**

core/src/main/java/hudson/PluginManager.java

@@ -600,6 +600,25 @@ public abstract class PluginManager extends AbstractModelObject implements OnMas
         }});
     }
 
+    void considerDetachedPlugin(String shortName) {
+        if (new File(rootDir, shortName + ".jpi").isFile()) {
+            LOGGER.fine(() -> "not considering loading a detached dependency " + shortName + " as it is already on disk");
+            return;
+        }
+        LOGGER.fine(() -> "considering loading a detached dependency " + shortName);
+        for (String loadedFile : loadPluginsFromWar("/WEB-INF/detached-plugins", (dir, name) -> normalisePluginName(name).equals(shortName))) {
+            String loaded = normalisePluginName(loadedFile);
+            File arc = new File(rootDir, loaded + ".jpi");
+            LOGGER.info(() -> "Loading a detached plugin as a dependency: " + arc);
+            try {
+                plugins.add(strategy.createPluginWrapper(arc));
+            } catch (IOException e) {
+                failedPlugins.add(new FailedPlugin(arc.getName(), e));
+            }
+
+        }
+    }
+
     protected @Nonnull Set<String> loadPluginsFromWar(@Nonnull String fromPath) {
         return loadPluginsFromWar(fromPath, null);
     }

core/src/main/java/hudson/model/UpdateSite.java

@@ -143,6 +143,7 @@ public class UpdateSite {
      */
     private static final String signatureValidatorPrefix = "update site";
 
+    private static final Set<String> warnedMissing = Collections.synchronizedSet(new HashSet<>());
 
     public UpdateSite(String id, String url) {
         this.id = id;
@@ -1093,7 +1094,7 @@ public class UpdateSite {
                 VersionNumber requiredVersion = e.getValue() != null ? new VersionNumber(e.getValue()) : null;
                 Plugin depPlugin = Jenkins.getInstance().getUpdateCenter().getPlugin(e.getKey(), requiredVersion);
                 if (depPlugin == null) {
-                    LOGGER.log(Level.WARNING, "Could not find dependency {0} of {1}", new Object[] {e.getKey(), name});
+                    LOGGER.log(warnedMissing.add(e.getKey()) ? Level.WARNING : Level.FINE, "Could not find dependency {0} of {1}", new Object[] {e.getKey(), name});
                     continue;
                 }
 

test/src/test/java/hudson/model/AbstractProjectTest.java

@@ -476,9 +476,6 @@ public class AbstractProjectTest {
     public void configDotXmlSubmissionToDifferentType() throws Exception {
         TestPluginManager tpm = (TestPluginManager) j.jenkins.pluginManager;
         tpm.installDetachedPlugin("javadoc");
-        tpm.installDetachedPlugin("junit");
-        tpm.installDetachedPlugin("display-url-api");
-        tpm.installDetachedPlugin("mailer");
         tpm.installDetachedPlugin("maven-plugin");
 
         j.jenkins.setCrumbIssuer(null);

test/src/test/java/hudson/model/UsageStatisticsTest.java

@@ -24,7 +24,6 @@
 package hudson.model;
 
 import com.google.common.io.Resources;
-import hudson.ClassicPluginStrategy;
 import hudson.Util;
 import hudson.model.UsageStatistics.CombinedCipherInputStream;
 import hudson.node_monitors.ArchitectureMonitor;
@@ -118,7 +117,7 @@ public class UsageStatisticsTest {
             assertThat("No duplicates", reported.contains(name), is(false));
             reported.add(name);
         }
-        assertThat(reported, containsInAnyOrder("credentials"));
+        assertThat(reported, hasItem("credentials"));
 
         // Compare content to watch out for backwards compatibility
         compareWithFile("jobs.json", sortJobTypes((JSONObject) o.get("jobs")));

test/src/test/java/jenkins/install/LoadDetachedPluginsTest.java

@@ -24,6 +24,7 @@
 
 package jenkins.install;
 
+import hudson.ClassicPluginStrategy;
 import jenkins.plugins.DetachedPluginsUtil;
 import jenkins.plugins.DetachedPluginsUtil.DetachedPlugin;
 import hudson.Plugin;
@@ -34,6 +35,8 @@ import hudson.util.VersionNumber;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
 
 import org.junit.Rule;
 import org.junit.Test;
@@ -44,18 +47,15 @@ import org.jvnet.hudson.test.RestartableJenkinsRule;
 import org.jvnet.hudson.test.SmokeTest;
 import org.jvnet.hudson.test.recipes.LocalData;
 
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
+import static org.hamcrest.Matchers.*;
+import static org.junit.Assert.*;
+import org.jvnet.hudson.test.LoggerRule;
 
 @Category(SmokeTest.class)
 public class LoadDetachedPluginsTest {
 
     @Rule public RestartableJenkinsRule rr = PluginManagerUtil.newRestartableJenkinsRule();
+    @Rule public LoggerRule logging = new LoggerRule();
 
     @Issue("JENKINS-48365")
     @Test
@@ -119,7 +119,7 @@ public class LoadDetachedPluginsTest {
             Plugin scriptSecurity = r.jenkins.getPlugin("script-security");
             assertThat("Script-security should be installed", scriptSecurity, notNullValue());
             assertThat("Dependencies of detached plugins should be upgraded to the required version",
-                    scriptSecurity.getWrapper().getVersionNumber(), equalTo(new VersionNumber("1.18.1")));
+                    scriptSecurity.getWrapper().getVersionNumber(), equalTo(new VersionNumber("1.56")));
             assertNoFailedPlugins(r);
         });
     }
@@ -161,6 +161,29 @@ public class LoadDetachedPluginsTest {
         });
     }
 
+    @Issue("JENKINS-55582")
+    @LocalData
+    @Test
+    public void installDetachedDependencies() {
+        logging.record(PluginManager.class, Level.FINE).record(ClassicPluginStrategy.class, Level.FINE);
+        rr.then(r -> {
+            List<String> activePlugins = r.jenkins.getPluginManager().getPlugins().stream().filter(PluginWrapper::isActive).map(PluginWrapper::getShortName).collect(Collectors.toList());
+            assertThat("we precreated $JENKINS_HOME/plugins/example.jpi so it had better be loaded", activePlugins, hasItem("example"));
+            { // Check that it links correctly against an implied dependency from a detached plugin:
+                Class<?> callerC = r.jenkins.pluginManager.uberClassLoader.loadClass("io.jenkins.plugins.example.Caller");
+                assertLoader(callerC, "example", r);
+                Object jdkInstaller = callerC.getMethod("use").invoke(null);
+                assertLoader(jdkInstaller.getClass(), "jdk-tool", r);
+            }
+            assertThat("it had various implicit detached dependencies so those should have been loaded too", activePlugins, hasSize(greaterThan(1)));
+        });
+    }
+    private void assertLoader(Class<?> c, String expectedPlugin, JenkinsRule r) {
+        PluginWrapper pw = r.jenkins.pluginManager.whichPlugin(c);
+        assertNotNull("did not expect to be loading " + c + " from " + c.getClassLoader(), pw);
+        assertEquals(expectedPlugin, pw.getShortName());
+    }
+
     private List<PluginWrapper> getInstalledDetachedPlugins(JenkinsRule r, List<DetachedPlugin> detachedPlugins) {
         PluginManager pluginManager = r.jenkins.getPluginManager();
         List<PluginWrapper> installedPlugins = new ArrayList<>();

test/src/test/resources/jenkins/install/LoadDetachedPluginsTest/upgradeFromJenkins2WithNewerDependency/jenkins.install.InstallUtil.lastExecVersion

+2.0
\ No newline at end of file

test/src/test/resources/jenkins/install/LoadDetachedPluginsTest/upgradeFromJenkins2WithNewerPlugin/jenkins.install.InstallUtil.lastExecVersion

+2.0
\ No newline at end of file

test/src/test/resources/jenkins/install/LoadDetachedPluginsTest/upgradeFromJenkins2WithOlderDependency/jenkins.install.InstallUtil.lastExecVersion

+2.0
\ No newline at end of file

war/pom.xml

@@ -387,7 +387,7 @@ THE SOFTWARE.
                 <artifactItem>
                   <groupId>org.jenkins-ci.plugins</groupId>
                   <artifactId>script-security</artifactId>
-                  <version>1.18.1</version>
+                  <version>1.56</version>
                   <type>hpi</type>
                 </artifactItem>
                 <artifactItem>