提交 e0ed9a86 编写于 作者: S Sam Van Oort 提交者: Oleg Nenashev

Micro-optimizations to Plugin data to reduce minimum memory for Jenkins (#3654)

* Reduce memory footprint for the plugin info by presizing hashmaps to reduce waste for small maps and interning commonly duplicated plugin strings

* Replace duplicated empty string arrays with the one in MemoryReductionUtil

* Fix interning where strings are null

* Fix to deal with the fact that we directly mutate dependency fields for UpdateSite, sigh

* Reduce starting HashMap size for some hashmaps likely to be empty or tiny

* Cleanup per review comments

* Trim some extra waste out of the UpdateSite warnings

* Fix count of optional dependencies

* Move predicates for dependency matching to top-level static final field so they are not duplicated
上级 c6a4fc9a
......@@ -26,6 +26,7 @@ package hudson;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import hudson.Proc.LocalProc;
import hudson.model.Computer;
import jenkins.util.MemoryReductionUtil;
import hudson.util.QuotedStringTokenizer;
import jenkins.model.Jenkins;
import hudson.model.TaskListener;
......@@ -396,7 +397,7 @@ public abstract class Launcher {
*/
@Nonnull
public String[] envs() {
return envs != null ? envs.clone() : new String[0];
return envs != null ? envs.clone() : MemoryReductionUtil.EMPTY_STRING_ARRAY;
}
/**
......
......@@ -188,7 +188,7 @@ public class PluginWrapper implements Comparable<PluginWrapper>, ModelObject {
/**
* A String error message, and a boolean indicating whether it's an original error (false) or downstream from an original one (true)
*/
private final transient Map<String, Boolean> dependencyErrors = new HashMap<>();
private final transient Map<String, Boolean> dependencyErrors = new HashMap<>(0);
/**
* Is this plugin bundled in jenkins.war?
......@@ -256,8 +256,8 @@ public class PluginWrapper implements Comparable<PluginWrapper>, ModelObject {
int idx = s.indexOf(':');
if(idx==-1)
throw new IllegalArgumentException("Illegal dependency specifier "+s);
this.shortName = s.substring(0,idx);
String version = s.substring(idx+1);
this.shortName = Util.intern(s.substring(0,idx));
String version = Util.intern(s.substring(idx+1));
boolean isOptional = false;
String[] osgiProperties = version.split("[;]");
......@@ -300,7 +300,7 @@ public class PluginWrapper implements Comparable<PluginWrapper>, ModelObject {
List<Dependency> dependencies, List<Dependency> optionalDependencies) {
this.parent = parent;
this.manifest = manifest;
this.shortName = computeShortName(manifest, archive.getName());
this.shortName = Util.intern(computeShortName(manifest, archive.getName()));
this.baseResourceURL = baseResourceURL;
this.classLoader = classLoader;
this.disableFile = disableFile;
......
......@@ -26,6 +26,7 @@ package hudson;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import hudson.model.TaskListener;
import jenkins.util.MemoryReductionUtil;
import hudson.util.QuotedStringTokenizer;
import hudson.util.VariableResolver;
import jenkins.util.SystemProperties;
......@@ -92,7 +93,6 @@ import javax.annotation.Nullable;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
/**
......@@ -1388,7 +1388,7 @@ public class Util {
@Nonnull String symlinkPath, @Nonnull TaskListener listener) throws InterruptedException {
try {
Path path = fileToPath(new File(baseDir, symlinkPath));
Path target = Paths.get(targetPath, new String[0]);
Path target = Paths.get(targetPath, MemoryReductionUtil.EMPTY_STRING_ARRAY);
final int maxNumberOfTries = 4;
final int timeInMillis = 100;
......
......@@ -32,6 +32,7 @@ import hudson.Util;
import hudson.XmlFile;
import hudson.model.*;
import hudson.util.HttpResponses;
import jenkins.util.MemoryReductionUtil;
import jenkins.model.Jenkins;
import hudson.model.listeners.SaveableListener;
import hudson.remoting.Channel;
......@@ -140,7 +141,7 @@ public class LogRecorder extends AbstractModelObject implements Saveable {
candidateNames.retainAll(partCandidates);
}
AutoCompletionCandidates candidates = new AutoCompletionCandidates();
candidates.add(candidateNames.toArray(new String[0]));
candidates.add(candidateNames.toArray(MemoryReductionUtil.EMPTY_STRING_ARRAY));
return candidates;
}
......
......@@ -137,7 +137,7 @@ public abstract class Descriptor<T extends Describable<T>> implements Saveable,
*/
public transient final Class<? extends T> clazz;
private transient final Map<String,CheckMethod> checkMethods = new ConcurrentHashMap<String,CheckMethod>();
private transient final Map<String,CheckMethod> checkMethods = new ConcurrentHashMap<String,CheckMethod>(2);
/**
* Lazily computed list of properties on {@link #clazz} and on the descriptor itself.
......@@ -233,7 +233,7 @@ public abstract class Descriptor<T extends Describable<T>> implements Saveable,
*
* @see #getHelpFile(String)
*/
private transient final Map<String,HelpRedirect> helpRedirect = new HashMap<String,HelpRedirect>();
private transient final Map<String,HelpRedirect> helpRedirect = new HashMap<String,HelpRedirect>(2);
private static class HelpRedirect {
private final Class<? extends Describable> owner;
......
......@@ -35,6 +35,7 @@ import hudson.security.AccessControlled;
import hudson.triggers.Trigger;
import hudson.util.DescriptorList;
import hudson.util.EditDistance;
import jenkins.util.MemoryReductionUtil;
import hudson.util.XStream2;
import java.io.File;
import java.io.IOException;
......@@ -313,8 +314,8 @@ public class Items {
// Had difficulty adapting the version in Functions to use no live items, so rewrote it:
static String getRelativeNameFrom(String itemFullName, String groupFullName) {
String[] itemFullNameA = itemFullName.isEmpty() ? new String[0] : itemFullName.split("/");
String[] groupFullNameA = groupFullName.isEmpty() ? new String[0] : groupFullName.split("/");
String[] itemFullNameA = itemFullName.isEmpty() ? MemoryReductionUtil.EMPTY_STRING_ARRAY : itemFullName.split("/");
String[] groupFullNameA = groupFullName.isEmpty() ? MemoryReductionUtil.EMPTY_STRING_ARRAY : groupFullName.split("/");
for (int i = 0; ; i++) {
if (i == itemFullNameA.length) {
if (i == groupFullNameA.length) {
......
......@@ -35,6 +35,7 @@ import hudson.model.UpdateCenter.UpdateCenterJob;
import hudson.util.FormValidation;
import hudson.util.FormValidation.Kind;
import hudson.util.HttpResponses;
import static jenkins.util.MemoryReductionUtil.*;
import hudson.util.TextFile;
import static java.util.concurrent.TimeUnit.*;
import hudson.util.VersionNumber;
......@@ -46,7 +47,6 @@ import java.net.URLEncoder;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
......@@ -56,6 +56,7 @@ import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.function.Predicate;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
......@@ -527,7 +528,7 @@ public class UpdateSite {
public final String connectionCheckUrl;
Data(JSONObject o) {
this.sourceId = (String)o.get("id");
this.sourceId = Util.intern((String)o.get("id"));
JSONObject c = o.optJSONObject("core");
if (c!=null) {
core = new Entry(sourceId, c, url);
......@@ -557,7 +558,7 @@ public class UpdateSite {
}
}
}
plugins.put(e.getKey(), p);
plugins.put(Util.intern(e.getKey()), p);
}
connectionCheckUrl = (String)o.get("connectionCheckUrl");
......@@ -629,8 +630,8 @@ public class UpdateSite {
Entry(String sourceId, JSONObject o, String baseURL) {
this.sourceId = sourceId;
this.name = o.getString("name");
this.version = o.getString("version");
this.name = Util.intern(o.getString("name"));
this.version = Util.intern(o.getString("version"));
// Trim this to prevent issues when the other end used Base64.encodeBase64String that added newlines
// to the end in old commons-codec. Not the case on updates.jenkins-ci.org, but let's be safe.
......@@ -738,8 +739,8 @@ public class UpdateSite {
public WarningVersionRange(JSONObject o) {
this.name = Util.fixEmpty(o.optString("name"));
this.firstVersion = Util.fixEmpty(o.optString("firstVersion"));
this.lastVersion = Util.fixEmpty(o.optString("lastVersion"));
this.firstVersion = Util.intern(Util.fixEmpty(o.optString("firstVersion")));
this.lastVersion = Util.intern(Util.fixEmpty(o.optString("lastVersion")));
Pattern p;
try {
p = Pattern.compile(o.getString("pattern"));
......@@ -836,13 +837,13 @@ public class UpdateSite {
this.type = Type.UNKNOWN;
}
this.id = o.getString("id");
this.component = o.getString("name");
this.component = Util.intern(o.getString("name"));
this.message = o.getString("message");
this.url = o.getString("url");
if (o.has("versions")) {
List<WarningVersionRange> ranges = new ArrayList<>();
JSONArray versions = o.getJSONArray("versions");
List<WarningVersionRange> ranges = new ArrayList<>(versions.size());
for (int i = 0; i < versions.size(); i++) {
WarningVersionRange range = new WarningVersionRange(versions.getJSONObject(i));
ranges.add(range);
......@@ -926,6 +927,16 @@ public class UpdateSite {
}
}
private static String get(JSONObject o, String prop) {
if(o.has(prop))
return o.getString(prop);
else
return null;
}
static final Predicate<Object> IS_DEP_PREDICATE = x -> x instanceof JSONObject && get(((JSONObject)x), "name") != null;
static final Predicate<Object> IS_NOT_OPTIONAL = x-> "false".equals(get(((JSONObject)x), "optional"));
public final class Plugin extends Entry {
/**
* Optional URL to the Wiki page that discusses this plugin.
......@@ -967,13 +978,13 @@ public class UpdateSite {
* Dependencies of this plugin, a name -&gt; version mapping.
*/
@Exported
public final Map<String,String> dependencies = new HashMap<String,String>();
public final Map<String,String> dependencies;
/**
* Optional dependencies of this plugin.
*/
@Exported
public final Map<String,String> optionalDependencies = new HashMap<String,String>();
public final Map<String,String> optionalDependencies;
@DataBoundConstructor
public Plugin(String sourceId, JSONObject o) {
......@@ -981,30 +992,31 @@ public class UpdateSite {
this.wiki = get(o,"wiki");
this.title = get(o,"title");
this.excerpt = get(o,"excerpt");
this.compatibleSinceVersion = get(o,"compatibleSinceVersion");
this.requiredCore = get(o,"requiredCore");
this.categories = o.has("labels") ? (String[])o.getJSONArray("labels").toArray(new String[0]) : null;
this.compatibleSinceVersion = Util.intern(get(o,"compatibleSinceVersion"));
this.requiredCore = Util.intern(get(o,"requiredCore"));
this.categories = o.has("labels") ? internInPlace((String[])o.getJSONArray("labels").toArray(EMPTY_STRING_ARRAY)) : null;
JSONArray ja = o.getJSONArray("dependencies");
int depCount = (int)(ja.stream().filter(IS_DEP_PREDICATE.and(IS_NOT_OPTIONAL)).count());
int optionalDepCount = (int)(ja.stream().filter(IS_DEP_PREDICATE.and(IS_NOT_OPTIONAL.negate())).count());
dependencies = getPresizedMutableMap(depCount);
optionalDependencies = getPresizedMutableMap(optionalDepCount);
for(Object jo : o.getJSONArray("dependencies")) {
JSONObject depObj = (JSONObject) jo;
// Make sure there's a name attribute and that the optional value isn't true.
if (get(depObj,"name")!=null) {
String depName = Util.intern(get(depObj,"name"));
if (depName!=null) {
if (get(depObj, "optional").equals("false")) {
dependencies.put(get(depObj, "name"), get(depObj, "version"));
dependencies.put(depName, Util.intern(get(depObj, "version")));
} else {
optionalDependencies.put(get(depObj, "name"), get(depObj, "version"));
optionalDependencies.put(depName, Util.intern(get(depObj, "version")));
}
}
}
}
private String get(JSONObject o, String prop) {
if(o.has(prop))
return o.getString(prop);
else
return null;
}
public String getDisplayName() {
String displayName;
......
......@@ -42,6 +42,7 @@ import java.util.logging.Logger;
import javax.servlet.ServletException;
import jenkins.util.MemoryReductionUtil;
import jenkins.model.Jenkins;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
......@@ -325,10 +326,8 @@ public class Search implements StaplerProxy {
static final class TokenList {
private final String[] tokens;
private final static String[] EMPTY = new String[0];
public TokenList(String tokenList) {
tokens = tokenList!=null ? tokenList.split("(?<=\\s)(?=\\S)") : EMPTY;
tokens = tokenList!=null ? tokenList.split("(?<=\\s)(?=\\S)") : MemoryReductionUtil.EMPTY_STRING_ARRAY;
}
public int length() { return tokens.length; }
......
......@@ -8,6 +8,7 @@ import hudson.model.UnprotectedRootAction;
import java.util.ArrayList;
import java.util.List;
import jenkins.util.MemoryReductionUtil;
import jenkins.model.Jenkins;
import org.acegisecurity.Authentication;
......@@ -62,7 +63,7 @@ public class WhoAmI implements UnprotectedRootAction {
@Exported
public String[] getAuthorities() {
if (auth().getAuthorities() == null) {
return new String[0];
return MemoryReductionUtil.EMPTY_STRING_ARRAY;
}
List <String> authorities = new ArrayList<String>();
for (GrantedAuthority a : auth().getAuthorities()) {
......
......@@ -40,6 +40,8 @@ import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.CheckForNull;
import jenkins.util.MemoryReductionUtil;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
......@@ -188,7 +190,7 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
String[] kids = dir.list();
if (kids == null) {
// the job may have just been created
kids = EMPTY_STRING_ARRAY;
kids = MemoryReductionUtil.EMPTY_STRING_ARRAY;
}
SortedIntList list = new SortedIntList(kids.length / 2);
for (String s : kids) {
......@@ -586,8 +588,6 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
ASC, DESC, EXACT
}
private static final String[] EMPTY_STRING_ARRAY = new String[0];
private static final SortedMap EMPTY_SORTED_MAP = Collections.unmodifiableSortedMap(new TreeMap());
static final Logger LOGGER = Logger.getLogger(AbstractLazyLoadRunMap.class.getName());
......
......@@ -17,6 +17,7 @@
/* Copied from commons-validator:commons-validator:1.6, with [PATCH] modifications */
package jenkins.org.apache.commons.validator.routines;
import jenkins.util.MemoryReductionUtil;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
......@@ -72,8 +73,6 @@ public class DomainValidator implements Serializable {
private static final int MAX_DOMAIN_LENGTH = 253;
private static final String[] EMPTY_STRING_ARRAY = new String[0];
private static final long serialVersionUID = -4407125112880174009L;
// Regular expression strings for hostnames (derived from RFC2396 and RFC 1123)
......@@ -1853,16 +1852,16 @@ public class DomainValidator implements Serializable {
* using the getInstance methods which are all (now) synchronised.
*/
// WARNING: this array MUST be sorted, otherwise it cannot be searched reliably using binary search
private static volatile String[] countryCodeTLDsPlus = EMPTY_STRING_ARRAY;
private static volatile String[] countryCodeTLDsPlus = MemoryReductionUtil.EMPTY_STRING_ARRAY;
// WARNING: this array MUST be sorted, otherwise it cannot be searched reliably using binary search
private static volatile String[] genericTLDsPlus = EMPTY_STRING_ARRAY;
private static volatile String[] genericTLDsPlus = MemoryReductionUtil.EMPTY_STRING_ARRAY;
// WARNING: this array MUST be sorted, otherwise it cannot be searched reliably using binary search
private static volatile String[] countryCodeTLDsMinus = EMPTY_STRING_ARRAY;
private static volatile String[] countryCodeTLDsMinus = MemoryReductionUtil.EMPTY_STRING_ARRAY;
// WARNING: this array MUST be sorted, otherwise it cannot be searched reliably using binary search
private static volatile String[] genericTLDsMinus = EMPTY_STRING_ARRAY;
private static volatile String[] genericTLDsMinus = MemoryReductionUtil.EMPTY_STRING_ARRAY;
/**
* enum used by {@link DomainValidator#updateTLDOverride(DomainValidator.ArrayType, String[])}
......@@ -1893,10 +1892,10 @@ public class DomainValidator implements Serializable {
// For use by unit test code only
static synchronized void clearTLDOverrides() {
inUse = false;
countryCodeTLDsPlus = EMPTY_STRING_ARRAY;
countryCodeTLDsMinus = EMPTY_STRING_ARRAY;
genericTLDsPlus = EMPTY_STRING_ARRAY;
genericTLDsMinus = EMPTY_STRING_ARRAY;
countryCodeTLDsPlus = MemoryReductionUtil.EMPTY_STRING_ARRAY;
countryCodeTLDsMinus = MemoryReductionUtil.EMPTY_STRING_ARRAY;
genericTLDsPlus = MemoryReductionUtil.EMPTY_STRING_ARRAY;
genericTLDsMinus = MemoryReductionUtil.EMPTY_STRING_ARRAY;
}
/**
* Update one of the TLD override arrays.
......
/*
* The MIT License
*
* Copyright (c) 2018, CloudBees, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package jenkins.util;
import hudson.Util;
import java.util.HashMap;
import java.util.Map;
/**
* Utilities to reduce memory footprint
* @author Sam Van Oort
*/
public class MemoryReductionUtil {
/** Returns the capacity we need to allocate for a HashMap so it will hold all elements without needing to resize. */
public static int preallocatedHashmapCapacity(int elementsToHold) {
if (elementsToHold <= 0) {
return 0;
} else if (elementsToHold < 3) {
return elementsToHold+1;
} else {
return elementsToHold+elementsToHold/3; // Default load factor is 0.75, so we want to fill that much.
}
}
/** Returns a mutable HashMap presized to hold the given number of elements without needing to resize. */
public static Map getPresizedMutableMap(int elementCount) {
return new HashMap(preallocatedHashmapCapacity(elementCount));
}
/** Empty string array, exactly what it says on the tin. Avoids repeatedly created empty array when calling "toArray." */
public static final String[] EMPTY_STRING_ARRAY = new String[0];
/** Returns the input strings, but with all values interned. */
public static String[] internInPlace(String[] input) {
if (input == null) {
return null;
} else if (input.length == 0) {
return EMPTY_STRING_ARRAY;
}
for (int i=0; i<input.length; i++) {
input[i] = Util.intern(input[i]);
}
return input;
}
}
......@@ -54,6 +54,7 @@ import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import jenkins.MasterToSlaveFileCallable;
import jenkins.model.ArtifactManager;
import jenkins.security.MasterToSlaveCallable;
......@@ -175,7 +176,7 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
*/
@Deprecated
public @Nonnull String[] list(String glob) throws IOException {
return list(glob.replace('\\', '/'), null, true).toArray(new String[0]);
return list(glob.replace('\\', '/'), null, true).toArray(MemoryReductionUtil.EMPTY_STRING_ARRAY);
}
/**
......
......@@ -28,6 +28,8 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.TreeSet;
import java.util.stream.Collectors;
import jenkins.util.MemoryReductionUtil;
import org.apache.commons.io.IOUtils;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertThat;
......@@ -45,7 +47,7 @@ public class ClassFilterImplSanityTest {
public void whitelistSanity() throws Exception {
try (InputStream is = ClassFilterImpl.class.getResourceAsStream("whitelisted-classes.txt")) {
List<String> lines = IOUtils.readLines(is, StandardCharsets.UTF_8).stream().filter(line -> !line.matches("#.*|\\s*")).collect(Collectors.toList());
assertThat("whitelist is NOT ordered", new TreeSet<>(lines), contains(lines.toArray(new String[0])));
assertThat("whitelist is NOT ordered", new TreeSet<>(lines), contains(lines.toArray(MemoryReductionUtil.EMPTY_STRING_ARRAY)));
for (String line : lines) {
try {
Class.forName(line);
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册