[JENKINS-40252] Switch to allItems where traversal order is not important

- Also switch in cases where we have a subset that is likely significantly smaller and hence quicker to sort

core/src/main/java/hudson/DependencyRunner.java

@@ -59,7 +59,7 @@ public class DependencyRunner implements Runnable {
             Set<AbstractProject> topLevelProjects = new HashSet<AbstractProject>();
             // Get all top-level projects
             LOGGER.fine("assembling top level projects");
-            for (AbstractProject p : Jenkins.getInstance().getAllItems(AbstractProject.class))
+            for (AbstractProject p : Jenkins.getInstance().allItems(AbstractProject.class))
                 if (p.getUpstreamProjects().size() == 0) {
                     LOGGER.fine("adding top level project " + p.getName());
                     topLevelProjects.add(p);

core/src/main/java/hudson/model/Computer.java

@@ -778,7 +778,7 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
     }
 
     public RunList getBuilds() {
-    	return new RunList(Jenkins.getInstance().getAllItems(Job.class)).node(getNode());
+        return RunList.fromJobs(Jenkins.getInstance().allItems(Job.class)).node(getNode());
     }
 
     /**

core/src/main/java/hudson/model/DependencyGraph.java

@@ -91,7 +91,7 @@ public class DependencyGraph implements Comparator<AbstractProject> {
         SecurityContext saveCtx = ACL.impersonate(ACL.SYSTEM);
         try {
             this.computationalData = new HashMap<Class<?>, Object>();
-            for( AbstractProject p : getAllProjects() )
+            for( AbstractProject p : Jenkins.getInstance().allItems(AbstractProject.class) )
                 p.buildDependencyGraph(this);
 
             forward = finalize(forward);
@@ -147,10 +147,6 @@ public class DependencyGraph implements Comparator<AbstractProject> {
         topologicallySorted = Collections.unmodifiableList(topologicallySorted);
     }
 
-    Collection<AbstractProject> getAllProjects() {
-        return Jenkins.getInstance().getAllItems(AbstractProject.class);
-    }
-
     /**
      * Special constructor for creating an empty graph
      */

core/src/main/java/hudson/model/Items.java

@@ -464,10 +464,9 @@ public class Items {
      * @since 1.538
      */
     public static @CheckForNull <T extends Item> T findNearest(Class<T> type, String name, ItemGroup context) {
-        List<T> projects = Jenkins.getInstance().getAllItems(type);
-        String[] names = new String[projects.size()];
-        for (int i = 0; i < projects.size(); i++) {
-            names[i] = projects.get(i).getRelativeNameFrom(context);
+        List<String> names = new ArrayList<>();
+        for (T item: Jenkins.getInstance().allItems(type)) {
+            names.add(item.getRelativeNameFrom(context));
         }
         String nearest = EditDistance.findNearest(name, names);
         return Jenkins.getInstance().getItem(nearest, context, type);

core/src/main/java/hudson/model/Label.java

@@ -362,10 +362,11 @@ public abstract class Label extends Actionable implements Comparable<Label>, Mod
     @Exported
     public List<AbstractProject> getTiedJobs() {
         List<AbstractProject> r = new ArrayList<AbstractProject>();
-        for (AbstractProject<?,?> p : Jenkins.getInstance().getAllItems(AbstractProject.class)) {
+        for (AbstractProject<?,?> p : Jenkins.getInstance().allItems(AbstractProject.class)) {
             if(p instanceof TopLevelItem && this.equals(p.getAssignedLabel()))
                 r.add(p);
         }
+        Collections.sort(r, Items.BY_FULL_NAME);
         return r;
     }
 

core/src/main/java/hudson/model/ListView.java

@@ -480,7 +480,7 @@ public class ListView extends View implements DirectlyModifiableView {
                     renameViewItem(oldFullName, newFullName, jenkins, (ListView) view);
                 }
             }
-            for (Item g : jenkins.getAllItems()) {
+            for (Item g : jenkins.allItems()) {
                 if (g instanceof ViewGroup) {
                     ViewGroup vg = (ViewGroup) g;
                     for (View v : vg.getViews()) {
@@ -524,7 +524,7 @@ public class ListView extends View implements DirectlyModifiableView {
                     deleteViewItem(item, jenkins, (ListView) view);
                 }
             }
-            for (Item g : jenkins.getAllItems()) {
+            for (Item g : jenkins.allItems()) {
                 if (g instanceof ViewGroup) {
                     ViewGroup vg = (ViewGroup) g;
                     for (View v : vg.getViews()) {

core/src/main/java/hudson/model/UsageStatistics.java

@@ -158,14 +158,22 @@ public class UsageStatistics extends PageDecorator {
         o.put("plugins",plugins);
 
         JSONObject jobs = new JSONObject();
-        List<TopLevelItem> items = j.getAllItems(TopLevelItem.class);
-        for (TopLevelItemDescriptor d : Items.all()) {
-            int cnt=0;
-            for (TopLevelItem item : items) {
-                if(item.getDescriptor()==d)
-                    cnt++;
+        // capture the descriptors as these should be small compared with the number of items
+        // so we will walk all items only once and we can short-cut the search of descriptors
+        TopLevelItemDescriptor[] descriptors = Items.all().toArray(new TopLevelItemDescriptor[0]);
+        int counts[] = new int[descriptors.length];
+        for (TopLevelItem item: j.allItems(TopLevelItem.class)) {
+            TopLevelItemDescriptor d = item.getDescriptor();
+            for (int i = 0; i < descriptors.length; i++) {
+                if (d == descriptors[i]) {
+                    counts[i]++;
+                    // no point checking any more, we found the match
+                    break;
+                }
             }
-            jobs.put(d.getJsonSafeClassName(),cnt);
+        }
+        for (int i = 0; i < descriptors.length; i++) {
+            jobs.put(descriptors[i].getJsonSafeClassName(), counts[i]);
         }
         o.put("jobs",jobs);
 

core/src/main/java/hudson/model/User.java

@@ -24,11 +24,16 @@
  */
 package hudson.model;
 
-import jenkins.security.UserDetailsCache;
-import jenkins.util.SystemProperties;
 import com.google.common.base.Predicate;
 import com.infradna.tool.bridge_method_injector.WithBridgeMethods;
-import hudson.*;
+import hudson.BulkChange;
+import hudson.CopyOnWrite;
+import hudson.Extension;
+import hudson.ExtensionList;
+import hudson.ExtensionPoint;
+import hudson.FeedAdapter;
+import hudson.Util;
+import hudson.XmlFile;
 import hudson.model.Descriptor.FormException;
 import hudson.model.listeners.SaveableListener;
 import hudson.security.ACL;
@@ -40,36 +45,9 @@ import hudson.util.FormApply;
 import hudson.util.FormValidation;
 import hudson.util.RunList;
 import hudson.util.XStream2;
-import jenkins.model.IdStrategy;
-import jenkins.model.Jenkins;
-import jenkins.model.ModelObjectWithContextMenu;
-import jenkins.security.ImpersonatingUserDetailsService;
-import jenkins.security.LastGrantedAuthoritiesProperty;
-import net.sf.json.JSONObject;
-
-import org.acegisecurity.Authentication;
-import org.acegisecurity.GrantedAuthority;
-import org.acegisecurity.providers.UsernamePasswordAuthenticationToken;
-import org.acegisecurity.providers.anonymous.AnonymousAuthenticationToken;
-import org.acegisecurity.userdetails.UserDetails;
-import org.acegisecurity.userdetails.UsernameNotFoundException;
-import org.jenkinsci.Symbol;
-import org.springframework.dao.DataAccessException;
-import org.kohsuke.accmod.Restricted;
-import org.kohsuke.accmod.restrictions.NoExternalUse;
-import org.kohsuke.stapler.StaplerRequest;
-import org.kohsuke.stapler.StaplerResponse;
-import org.kohsuke.stapler.export.Exported;
-import org.kohsuke.stapler.export.ExportedBean;
-import org.apache.commons.io.filefilter.DirectoryFileFilter;
-import org.kohsuke.stapler.interceptor.RequirePOST;
-
-import javax.annotation.concurrent.GuardedBy;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletResponse;
 import java.io.File;
-import java.io.IOException;
 import java.io.FileFilter;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -91,7 +69,34 @@ import java.util.logging.Logger;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import javax.annotation.concurrent.GuardedBy;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletResponse;
+import jenkins.model.IdStrategy;
+import jenkins.model.Jenkins;
+import jenkins.model.ModelObjectWithContextMenu;
+import jenkins.security.ImpersonatingUserDetailsService;
+import jenkins.security.LastGrantedAuthoritiesProperty;
+import jenkins.security.UserDetailsCache;
+import jenkins.util.SystemProperties;
+import net.sf.json.JSONObject;
+import org.acegisecurity.Authentication;
+import org.acegisecurity.GrantedAuthority;
+import org.acegisecurity.providers.UsernamePasswordAuthenticationToken;
+import org.acegisecurity.providers.anonymous.AnonymousAuthenticationToken;
+import org.acegisecurity.userdetails.UserDetails;
+import org.acegisecurity.userdetails.UsernameNotFoundException;
+import org.apache.commons.io.filefilter.DirectoryFileFilter;
 import org.apache.commons.lang.StringUtils;
+import org.jenkinsci.Symbol;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+import org.kohsuke.stapler.StaplerRequest;
+import org.kohsuke.stapler.StaplerResponse;
+import org.kohsuke.stapler.export.Exported;
+import org.kohsuke.stapler.export.ExportedBean;
+import org.kohsuke.stapler.interceptor.RequirePOST;
+import org.springframework.dao.DataAccessException;
 
 /**
  * Represents a user.
@@ -647,9 +652,10 @@ public class User extends AbstractModelObject implements AccessControlled, Descr
      * Gets the list of {@link Build}s that include changes by this user,
      * by the timestamp order.
      */
+    @SuppressWarnings("unchecked")
     @WithBridgeMethods(List.class)
     public @Nonnull RunList getBuilds() {
-    	return new RunList<Run<?,?>>(Jenkins.getInstance().getAllItems(Job.class)).filter(new Predicate<Run<?,?>>() {
+        return RunList.fromJobs(Jenkins.getInstance().allItems(Job.class)).filter(new Predicate<Run<?,?>>() {
             @Override public boolean apply(Run<?,?> r) {
                 return r instanceof AbstractBuild && relatedTo((AbstractBuild<?,?>) r);
             }
@@ -662,7 +668,7 @@ public class User extends AbstractModelObject implements AccessControlled, Descr
      */
     public @Nonnull Set<AbstractProject<?,?>> getProjects() {
         Set<AbstractProject<?,?>> r = new HashSet<AbstractProject<?,?>>();
-        for (AbstractProject<?,?> p : Jenkins.getInstance().getAllItems(AbstractProject.class))
+        for (AbstractProject<?,?> p : Jenkins.getInstance().allItems(AbstractProject.class))
             if(p.hasParticipant(this))
                 r.add(p);
         return r;
@@ -833,7 +839,7 @@ public class User extends AbstractModelObject implements AccessControlled, Descr
 
     public void doRssLatest(StaplerRequest req, StaplerResponse rsp) throws IOException, ServletException {
         final List<Run> lastBuilds = new ArrayList<Run>();
-        for (AbstractProject<?,?> p : Jenkins.getInstance().getAllItems(AbstractProject.class)) {
+        for (AbstractProject<?,?> p : Jenkins.getInstance().allItems(AbstractProject.class)) {
             for (AbstractBuild<?,?> b = p.getLastBuild(); b != null; b = b.getPreviousBuild()) {
                 if (relatedTo(b)) {
                     lastBuilds.add(b);
@@ -841,6 +847,14 @@ public class User extends AbstractModelObject implements AccessControlled, Descr
                 }
             }
         }
+        // historically these have been reported sorted by project name, we switched to the lazy iteration
+        // so we only have to sort the sublist of runs rather than the full list of irrelevant projects
+        Collections.sort(lastBuilds, new Comparator<Run>() {
+            @Override
+            public int compare(Run o1, Run o2) {
+                return Items.BY_FULL_NAME.compare(o1.getParent(), o2.getParent());
+            }
+        });
         rss(req, rsp, " latest build", RunList.fromRuns(lastBuilds), Run.FEED_ADAPTER_LATEST);
     }
 

core/src/main/java/hudson/model/WorkspaceCleanupThread.java

@@ -68,7 +68,7 @@ public class WorkspaceCleanupThread extends AsyncPeriodicWork {
         Jenkins j = Jenkins.getInstance();
         nodes.add(j);
         nodes.addAll(j.getNodes());
-        for (TopLevelItem item : j.getAllItems(TopLevelItem.class)) {
+        for (TopLevelItem item : j.allItems(TopLevelItem.class)) {
             if (item instanceof ModifiableTopLevelItemGroup) { // no such thing as TopLevelItemGroup, and ItemGroup offers no access to its type parameter
                 continue; // children will typically have their own workspaces as subdirectories; probably no real workspace of its own
             }

core/src/main/java/hudson/model/listeners/ItemListener.java

@@ -240,11 +240,7 @@ public class ItemListener implements ExtensionPoint {
             }
         });
         if (rootItem instanceof ItemGroup) {
-            for (final Item child : ACL.impersonate(ACL.SYSTEM, new NotReallyRoleSensitiveCallable<List<Item>,RuntimeException>() {
-                @Override public List<Item> call() {
-                    return Items.getAllItems((ItemGroup) rootItem, Item.class);
-                }
-            })) {
+            for (final Item child : Items.allItems(ACL.SYSTEM, (ItemGroup)rootItem, Item.class)) {
                 final String childNew = child.getFullName();
                 assert childNew.startsWith(newFullName);
                 assert childNew.charAt(newFullName.length()) == '/';

core/src/main/java/hudson/scm/AutoBrowserHolder.java

@@ -76,7 +76,7 @@ final class AutoBrowserHolder {
      *      null if no applicable configuration was found.
      */
     private RepositoryBrowser infer() {
-        for( AbstractProject p : Jenkins.getInstance().getAllItems(AbstractProject.class) ) {
+        for( AbstractProject p : Jenkins.getInstance().allItems(AbstractProject.class) ) {
             SCM scm = p.getScm();
             if (scm!=null && scm.getClass()==owner.getClass() && scm.getBrowser()!=null &&
                     ((SCMDescriptor)scm.getDescriptor()).isBrowserReusable(scm,owner)) {

core/src/main/java/hudson/search/CollectionSearchIndex.java

@@ -24,8 +24,10 @@
 package hudson.search;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import javax.annotation.Nonnull;
 
 /**
  * {@link SearchIndex} built on a {@link Map}.
@@ -44,6 +46,12 @@ public abstract class CollectionSearchIndex<SMT extends SearchableModelObject> i
      */
     protected abstract Collection<SMT> all();
 
+    @Nonnull
+    protected Iterable<SMT> allAsIterable() {
+        Collection<SMT> all = all();
+        return all == null ? Collections.<SMT>emptySet() : all;
+    }
+
     public void find(String token, List<SearchItem> result) {
         SearchItem p = get(token);
         if(p!=null)
@@ -51,13 +59,11 @@ public abstract class CollectionSearchIndex<SMT extends SearchableModelObject> i
     }
 
     public void suggest(String token, List<SearchItem> result) {
-         Collection<SMT> items = all();
         boolean isCaseSensitive = UserSearchProperty.isCaseInsensitive();
         if(isCaseSensitive){
           token = token.toLowerCase();
         }
-        if(items==null)     return;
-        for (SMT o : items) {
+        for (SMT o : allAsIterable()) {
             String name = getName(o);
             if(isCaseSensitive)
                 name=name.toLowerCase();

core/src/main/java/hudson/tasks/ArtifactArchiver.java

@@ -348,7 +348,7 @@ public class ArtifactArchiver extends Recorder implements SimpleBuildStep {
     @Extension public static final class Migrator extends ItemListener {
         @SuppressWarnings("deprecation")
         @Override public void onLoaded() {
-            for (AbstractProject<?,?> p : Jenkins.getInstance().getAllItems(AbstractProject.class)) {
+            for (AbstractProject<?,?> p : Jenkins.getInstance().allItems(AbstractProject.class)) {
                 try {
                     ArtifactArchiver aa = p.getPublishersList().get(ArtifactArchiver.class);
                     if (aa != null && aa.latestOnly != null) {

core/src/main/java/hudson/tasks/BuildTrigger.java

@@ -420,7 +420,7 @@ public class BuildTrigger extends Recorder implements DependencyDeclarer {
             private void locationChanged(Item item, String oldFullName, String newFullName) {
                 // update BuildTrigger of other projects that point to this object.
                 // can't we generalize this?
-                for( Project<?,?> p : Jenkins.getInstance().getAllItems(Project.class) ) {
+                for( Project<?,?> p : Jenkins.getInstance().allItems(Project.class) ) {
                     BuildTrigger t = p.getPublishersList().get(BuildTrigger.class);
                     if(t!=null) {
                         String cp2 = Items.computeRelativeNamesAfterRenaming(oldFullName, newFullName, t.childProjects, p.getParent());

core/src/main/java/hudson/triggers/SCMTrigger.java

@@ -310,12 +310,24 @@ public class SCMTrigger extends Trigger<Item> {
 
         @Restricted(NoExternalUse.class)
         public boolean isPollingThreadCountOptionVisible() {
+            if (getPollingThreadCount() != 0) {
+                // this is a user who already configured the option
+                return true;
+            }
             // unless you have a fair number of projects, this option is likely pointless.
             // so let's hide this option for new users to avoid confusing them
             // unless it was already changed
-            // TODO switch to check for SCMTriggerItem
-            return Jenkins.getInstance().getAllItems(AbstractProject.class).size() > 10
-                    || getPollingThreadCount() != 0;
+            int count = 0;
+            // we are faster walking some items with a lazy iterator than building a list of all items just to query
+            // the size. This also lets us check against SCMTriggerItem rather than AbstractProject
+            for (Item item: Jenkins.getInstance().allItems(Item.class)) {
+                if (item instanceof SCMTriggerItem) {
+                    if (++count > 10) {
+                        return true;
+                    }
+                }
+            }
+            return false;
         }
 
         /**

core/src/main/java/hudson/triggers/Trigger.java

@@ -266,7 +266,7 @@ public abstract class Trigger<J extends Item> implements Describable<Trigger<?>>
         }
 
         // Process all triggers, except SCMTriggers when synchronousPolling is set
-        for (ParameterizedJobMixIn.ParameterizedJob p : inst.getAllItems(ParameterizedJobMixIn.ParameterizedJob.class)) {
+        for (ParameterizedJobMixIn.ParameterizedJob p : inst.allItems(ParameterizedJobMixIn.ParameterizedJob.class)) {
             for (Trigger t : p.getTriggers().values()) {
                 if (!(t instanceof SCMTrigger && scmd.synchronousPolling)) {
                     if (t !=null && t.spec != null && t.tabs != null) {

core/src/main/java/hudson/util/RunList.java

@@ -76,7 +76,14 @@ public class RunList<R extends Run> extends AbstractList<R> {
         this.base = combine(runLists);
     }
 
-    private Iterable<R> combine(Iterable<Iterable<R>> runLists) {
+    public static <J extends Job<J,R>, R extends Run<J,R>> RunList<R> fromJobs(Iterable<? extends J> jobs) {
+        List<Iterable<R>> runLists = new ArrayList<>();
+        for (Job j : jobs)
+            runLists.add(j.getBuilds());
+        return new RunList<>(combine(runLists));
+    }
+
+    private static <R extends Run> Iterable<R> combine(Iterable<Iterable<R>> runLists) {
         return Iterables.mergeSorted(runLists, new Comparator<R>() {
             public int compare(R o1, R o2) {
                 long lhs = o1.getTimeInMillis();

core/src/main/java/jenkins/model/Jenkins.java

@@ -1753,6 +1753,16 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
         return Items.getAllItems(this, type);
     }
 
+    /**
+     * Gets all the {@link Item}s unordered, lazily and recursively in the {@link ItemGroup} tree
+     * and filter them by the given type.
+     *
+     * @since FIXME
+     */
+    public <T extends Item> Iterable<T> allItems(Class<T> type) {
+        return Items.allItems(this, type);
+    }
+
     /**
      * Gets all the items recursively.
      *
@@ -1762,6 +1772,15 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
         return getAllItems(Item.class);
     }
 
+    /**
+     * Gets all the items unordered, lazily and recursively.
+     *
+     * @since FIXME
+     */
+    public Iterable<Item> allItems() {
+        return allItems(Item.class);
+    }
+
     /**
      * Gets a list of simple top-level projects.
      * @deprecated This method will ignore Maven and matrix projects, as well as projects inside containers such as folders.
@@ -1781,8 +1800,9 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
      */
     public Collection<String> getJobNames() {
         List<String> names = new ArrayList<String>();
-        for (Job j : getAllItems(Job.class))
+        for (Job j : allItems(Job.class))
             names.add(j.getFullName());
+        Collections.sort(names, String.CASE_INSENSITIVE_ORDER);
         return names;
     }
 
@@ -2248,6 +2268,11 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
             .add(new CollectionSearchIndex<TopLevelItem>() {
                 protected SearchItem get(String key) { return getItemByFullName(key, TopLevelItem.class); }
                 protected Collection<TopLevelItem> all() { return getAllItems(TopLevelItem.class); }
+                @Nonnull
+                @Override
+                protected Iterable<TopLevelItem> allAsIterable() {
+                    return allItems(TopLevelItem.class);
+                }
             })
             .add(getPrimaryView().makeSearchIndex())
             .add(new CollectionSearchIndex() {// for computers

core/src/main/java/jenkins/triggers/ReverseBuildTrigger.java

@@ -216,7 +216,7 @@ public final class ReverseBuildTrigger extends Trigger<Job> implements Dependenc
         private Map<Job,Collection<ReverseBuildTrigger>> calculateCache() {
             try (ACLContext _ = ACL.as(ACL.SYSTEM)) {
                 final Map<Job, Collection<ReverseBuildTrigger>> result = new WeakHashMap<>();
-                for (Job<?, ?> downstream : Jenkins.getInstance().getAllItems(Job.class)) {
+                for (Job<?, ?> downstream : Jenkins.getInstance().allItems(Job.class)) {
                     ReverseBuildTrigger trigger =
                             ParameterizedJobMixIn.getTrigger(downstream, ReverseBuildTrigger.class);
                     if (trigger == null) {
@@ -276,7 +276,7 @@ public final class ReverseBuildTrigger extends Trigger<Job> implements Dependenc
         @Override
         public void onLocationChanged(Item item, final String oldFullName, final String newFullName) {
             try (ACLContext _ = ACL.as(ACL.SYSTEM)) {
-                for (Job<?, ?> p : Jenkins.getInstance().getAllItems(Job.class)) {
+                for (Job<?, ?> p : Jenkins.getInstance().allItems(Job.class)) {
                     ReverseBuildTrigger t = ParameterizedJobMixIn.getTrigger(p, ReverseBuildTrigger.class);
                     if (t != null) {
                         String revised =