未验证 提交 51cac7de 编写于 作者: J Jesse Glick 提交者: GitHub

Merge pull request #3302 from jglick/VirtualFile-JENKINS-49635

[JEP-202] Extending VirtualFile
......@@ -139,6 +139,7 @@ import static hudson.Util.isSymlink;
import java.util.Collections;
import org.apache.tools.ant.BuildException;
import org.kohsuke.accmod.restrictions.Beta;
/**
* {@link File} like object with remoting support.
......@@ -903,6 +904,28 @@ public final class FilePath implements Serializable {
}
}
/**
* Copies the content of a URL to a remote file.
* Unlike {@link #copyFrom} this will not transfer content over a Remoting channel.
* @since FIXME
*/
@Restricted(Beta.class)
public void copyFromRemotely(URL url) throws IOException, InterruptedException {
act(new CopyFromRemotely(url));
}
private final class CopyFromRemotely extends MasterToSlaveFileCallable<Void> {
private static final long serialVersionUID = 1;
private final URL url;
CopyFromRemotely(URL url) {
this.url = url;
}
@Override
public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException {
copyFrom(url);
return null;
}
}
/**
* Replaces the content of this file by the data from the given {@link InputStream}.
*
......
......@@ -1420,7 +1420,7 @@ public class Util {
* The relative path is meant to be resolved from the location of the symlink.
*/
@CheckForNull
public static String resolveSymlink(@Nonnull File link) throws InterruptedException, IOException {
public static String resolveSymlink(@Nonnull File link) throws IOException {
try {
Path path = fileToPath(link);
return Files.readSymbolicLink(path).toString();
......
......@@ -29,9 +29,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.net.URL;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
......@@ -51,6 +53,7 @@ import org.apache.tools.zip.ZipOutputStream;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.HttpResponse;
import org.kohsuke.stapler.HttpResponses;
import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;
......@@ -213,7 +216,7 @@ public final class DirectoryBrowserSupport implements HttpResponse {
String rest = _rest.toString();
// this is the base file/directory
VirtualFile baseFile = root.child(base);
VirtualFile baseFile = base.isEmpty() ? root : root.child(base);
if(baseFile.isDirectory()) {
if(zip) {
......@@ -295,6 +298,14 @@ public final class DirectoryBrowserSupport implements HttpResponse {
return;
}
URL external = baseFile.toExternalURL();
if (external != null) {
// or this URL could be emitted directly from dir.jelly
// though we would prefer to delay toExternalURL calls unless and until needed
rsp.sendRedirect2(external.toExternalForm());
return;
}
long lastModified = baseFile.lastModified();
long length = baseFile.length();
......@@ -355,7 +366,8 @@ public final class DirectoryBrowserSupport implements HttpResponse {
private static void zip(OutputStream outputStream, VirtualFile dir, String glob) throws IOException {
try (ZipOutputStream zos = new ZipOutputStream(outputStream)) {
zos.setEncoding(System.getProperty("file.encoding")); // TODO JENKINS-20663 make this overridable via query parameter
for (String n : dir.list(glob.length() == 0 ? "**" : glob)) {
// TODO consider using run(Callable) here
for (String n : dir.list(glob.isEmpty() ? "**" : glob, null, /* TODO what is the user expectation? */true)) {
String relativePath;
if (glob.length() == 0) {
// JENKINS-19947: traditional behavior is to prepend the directory name
......@@ -535,10 +547,10 @@ public final class DirectoryBrowserSupport implements HttpResponse {
* @param baseRef String like "../../../" that cancels the 'rest' portion. Can be "./"
*/
private static List<List<Path>> patternScan(VirtualFile baseDir, String pattern, String baseRef) throws IOException {
String[] files = baseDir.list(pattern);
Collection<String> files = baseDir.list(pattern, null, /* TODO what is the user expectation? */true);
if (files.length > 0) {
List<List<Path>> r = new ArrayList<List<Path>>(files.length);
if (!files.isEmpty()) {
List<List<Path>> r = new ArrayList<List<Path>>(files.size());
for (String match : files) {
List<Path> file = buildPathList(baseDir, baseDir.child(match), baseRef);
r.add(file);
......
......@@ -73,6 +73,7 @@ import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.io.Reader;
import java.io.Serializable;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
......@@ -108,6 +109,7 @@ import jenkins.model.RunAction2;
import jenkins.model.StandardArtifactManager;
import jenkins.model.lazy.BuildReference;
import jenkins.model.lazy.LazyBuildMixIn;
import jenkins.security.MasterToSlaveCallable;
import jenkins.util.VirtualFile;
import jenkins.util.io.OnMaster;
import net.sf.json.JSONObject;
......@@ -1090,12 +1092,16 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
* @return The list can be empty but never null
*/
public @Nonnull List<Artifact> getArtifactsUpTo(int artifactsNumber) {
ArtifactList r = new ArtifactList();
SerializableArtifactList sal;
VirtualFile root = getArtifactManager().root();
try {
addArtifacts(getArtifactManager().root(), "", "", r, null, artifactsNumber);
sal = root.run(new AddArtifacts(root, artifactsNumber));
} catch (IOException x) {
LOGGER.log(Level.WARNING, null, x);
sal = new SerializableArtifactList();
}
ArtifactList r = new ArtifactList();
r.updateFrom(sal);
r.computeDisplayName();
return r;
}
......@@ -1109,9 +1115,25 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
return !getArtifactsUpTo(1).isEmpty();
}
private int addArtifacts(@Nonnull VirtualFile dir,
private static final class AddArtifacts extends MasterToSlaveCallable<SerializableArtifactList, IOException> {
private static final long serialVersionUID = 1L;
private final VirtualFile root;
private final int artifactsNumber;
AddArtifacts(VirtualFile root, int artifactsNumber) {
this.root = root;
this.artifactsNumber = artifactsNumber;
}
@Override
public SerializableArtifactList call() throws IOException {
SerializableArtifactList sal = new SerializableArtifactList();
addArtifacts(root, "", "", sal, null, artifactsNumber);
return sal;
}
}
private static int addArtifacts(@Nonnull VirtualFile dir,
@Nonnull String path, @Nonnull String pathHref,
@Nonnull ArtifactList r, @Nonnull Artifact parent, int upTo) throws IOException {
@Nonnull SerializableArtifactList r, @Nonnull SerializableArtifact parent, int upTo) throws IOException {
VirtualFile[] kids = dir.list();
Arrays.sort(kids);
......@@ -1122,26 +1144,26 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
String childHref = pathHref + Util.rawEncode(child);
String length = sub.isFile() ? String.valueOf(sub.length()) : "";
boolean collapsed = (kids.length==1 && parent!=null);
Artifact a;
SerializableArtifact a;
if (collapsed) {
// Collapse single items into parent node where possible:
a = new Artifact(parent.getFileName() + '/' + child, childPath,
a = new SerializableArtifact(parent.name + '/' + child, childPath,
sub.isDirectory() ? null : childHref, length,
parent.getTreeNodeId());
parent.treeNodeId);
r.tree.put(a, r.tree.remove(parent));
} else {
// Use null href for a directory:
a = new Artifact(child, childPath,
a = new SerializableArtifact(child, childPath,
sub.isDirectory() ? null : childHref, length,
"n" + ++r.idSeq);
r.tree.put(a, parent!=null ? parent.getTreeNodeId() : null);
r.tree.put(a, parent!=null ? parent.treeNodeId : null);
}
if (sub.isDirectory()) {
n += addArtifacts(sub, childPath + '/', childHref + '/', r, a, upTo-n);
if (n>=upTo) break;
} else {
// Don't store collapsed path in ArrayList (for correct data in external API)
r.add(collapsed ? new Artifact(child, a.relativePath, a.href, length, a.treeNodeId) : a);
r.add(collapsed ? new SerializableArtifact(child, a.relativePath, a.href, length, a.treeNodeId) : a);
if (++n>=upTo) break;
}
}
......@@ -1159,6 +1181,30 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
public static final int TREE_CUTOFF = Integer.parseInt(SystemProperties.getString("hudson.model.Run.ArtifactList.treeCutoff", "40"));
// ..and then "too many"
/** {@link Run.Artifact} without the implicit link to {@link Run} */
private static final class SerializableArtifact implements Serializable {
private static final long serialVersionUID = 1L;
final String name;
final String relativePath;
final String href;
final String length;
final String treeNodeId;
SerializableArtifact(String name, String relativePath, String href, String length, String treeNodeId) {
this.name = name;
this.relativePath = relativePath;
this.href = href;
this.length = length;
this.treeNodeId = treeNodeId;
}
}
/** {@link Run.ArtifactList} without the implicit link to {@link Run} */
private static final class SerializableArtifactList extends ArrayList<SerializableArtifact> {
private static final long serialVersionUID = 1L;
private LinkedHashMap<SerializableArtifact, String> tree = new LinkedHashMap<>();
private int idSeq = 0;
}
public final class ArtifactList extends ArrayList<Artifact> {
private static final long serialVersionUID = 1L;
......@@ -1167,7 +1213,24 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
* Contains Artifact objects for directories and files (the ArrayList contains only files).
*/
private LinkedHashMap<Artifact,String> tree = new LinkedHashMap<Artifact,String>();
private int idSeq = 0;
void updateFrom(SerializableArtifactList clone) {
Map<String, Artifact> artifacts = new HashMap<>(); // need to share objects between tree and list, since computeDisplayName mutates displayPath
for (SerializableArtifact sa : clone) {
Artifact a = new Artifact(sa);
artifacts.put(a.relativePath, a);
add(a);
}
tree = new LinkedHashMap<>();
for (Map.Entry<SerializableArtifact, String> entry : clone.tree.entrySet()) {
SerializableArtifact sa = entry.getKey();
Artifact a = artifacts.get(sa.relativePath);
if (a == null) {
a = new Artifact(sa);
}
tree.put(a, entry.getValue());
}
}
public Map<Artifact,String> getTree() {
return tree;
......@@ -1282,6 +1345,10 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
*/
private String length;
Artifact(SerializableArtifact clone) {
this(clone.name, clone.relativePath, clone.href, clone.length, clone.treeNodeId);
}
/*package for test*/ Artifact(String name, String relativePath, String href, String len, String treeNodeId) {
this.name = name;
this.relativePath = relativePath;
......
......@@ -7,7 +7,6 @@ import org.apache.tools.ant.types.FileSet;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.Serializable;
import static hudson.Util.fixEmpty;
......@@ -31,19 +30,15 @@ public abstract class DirScanner implements Serializable {
*/
protected final void scanSingle(File f, String relative, FileVisitor visitor) throws IOException {
if (visitor.understandsSymlink()) {
String target;
try {
String target;
try {
target = Util.resolveSymlink(f);
} catch (IOException x) { // JENKINS-13202
target = null;
}
if (target != null) {
visitor.visitSymlink(f, target, relative);
return;
}
} catch (InterruptedException e) {
throw (IOException) new InterruptedIOException().initCause(e);
target = Util.resolveSymlink(f);
} catch (IOException x) { // JENKINS-13202
target = null;
}
if (target != null) {
visitor.visitSymlink(f, target, relative);
return;
}
}
visitor.visit(f, relative);
......
......@@ -117,7 +117,7 @@ public class IOUtils {
* execute permissions for the owner, group, and others, i.e. the max return value
* is 0777. Consider using {@link Files#getPosixFilePermissions} instead if you only
* care about access permissions.
*
* <p>If the file is symlink, the mode is that of the link target, not the link itself.
* @return a file mode, or -1 if not on Unix
* @throws PosixException if the file could not be statted, e.g. broken symlink
*/
......
......@@ -25,29 +25,45 @@
package jenkins.util;
import hudson.FilePath;
import hudson.Util;
import hudson.model.DirectoryBrowserSupport;
import hudson.os.PosixException;
import hudson.remoting.Callable;
import hudson.remoting.Channel;
import hudson.remoting.RemoteInputStream;
import hudson.remoting.VirtualChannel;
import hudson.util.DirScanner;
import hudson.util.FileVisitor;
import hudson.util.IOUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.LinkOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import jenkins.MasterToSlaveFileCallable;
import jenkins.model.ArtifactManager;
import jenkins.security.MasterToSlaveCallable;
import org.apache.tools.ant.DirectoryScanner;
import org.apache.tools.ant.types.AbstractFileSet;
import org.apache.tools.ant.types.selectors.SelectorUtils;
import org.apache.tools.ant.types.selectors.TokenizedPath;
import org.apache.tools.ant.types.selectors.TokenizedPattern;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.Beta;
/**
* Abstraction over {@link File}, {@link FilePath}, or other items such as network resources or ZIP entries.
......@@ -62,6 +78,25 @@ import jenkins.MasterToSlaveFileCallable;
* {@link VirtualFile} makes no assumption about where the actual files are, or whether there really exists
* {@link File}s somewhere. This makes VirtualFile more abstract.
*
* <h2>Opening files from other machines</h2>
*
* While {@link VirtualFile} is marked {@link Serializable},
* it is <em>not</em> safe in general to transfer over a Remoting channel.
* (For example, an implementation from {@link #forFilePath} could be sent on the <em>same</em> channel,
* but an implementation from {@link #forFile} will not.)
* Thus callers should assume that methods such as {@link #open} will work
* only on the node on which the object was created.
*
* <p>Since some implementations may in fact use external file storage,
* callers may request optional APIs to access those services more efficiently.
* Otherwise, for example, a plugin copying a file
* previously saved by {@link ArtifactManager} to an external storage service
* which tunneled a stream from {@link #open} using {@link RemoteInputStream}
* would wind up transferring the file from the service to the Jenkins master and then on to an agent.
* Similarly, if {@link DirectoryBrowserSupport} rendered a link to an in-Jenkins URL,
* a large file could be transferred from the service to the Jenkins master and then on to the browser.
* To avoid this overhead, callers may check whether an implementation supports {@link #toExternalURL}.
*
* @see DirectoryBrowserSupport
* @see FilePath
* @since 1.532
......@@ -80,9 +115,11 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
/**
* Gets a URI.
* Should at least uniquely identify this virtual file within its root, but not necessarily globally.
* <p>When {@link #toExternalURL} is implemented, that same value could be used here,
* unless some sort of authentication is also embedded.
* @return a URI (need not be absolute)
*/
public abstract URI toURI();
public abstract @Nonnull URI toURI();
/**
* Gets the parent file.
......@@ -105,8 +142,22 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
*/
public abstract boolean isFile() throws IOException;
/**
* If this file is a symlink, returns the link target.
* <p>The default implementation always returns null.
* Some implementations may not support symlinks under any conditions.
* @return a target (typically a relative path in some format), or null if this is not a link
* @throws IOException if reading the link, or even determining whether this file is a link, failed
* @since FIXME
*/
@Restricted(Beta.class)
public @CheckForNull String readLink() throws IOException {
return null;
}
/**
* Checks whether this file exists.
* The behavior is undefined for symlinks; if in doubt, check {@link #readLink} first.
* @return true if it is a plain file or directory, false if nonexistent
* @throws IOException in case checking status failed
*/
......@@ -119,13 +170,75 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
*/
public abstract @Nonnull VirtualFile[] list() throws IOException;
/**
* @deprecated use {@link #list(String, String, boolean)} instead
*/
@Deprecated
public @Nonnull String[] list(String glob) throws IOException {
return list(glob.replace('\\', '/'), null, true).toArray(new String[0]);
}
/**
* Lists recursive files of this directory with pattern matching.
* @param glob an Ant-style glob
* @return a list of relative names of children (files directly inside or in subdirectories)
* <p>The default implementation calls {@link #list()} recursively inside {@link #run} and applies filtering to the result.
* Implementations may wish to override this more efficiently.
* @param includes comma-separated Ant-style globs as per {@link Util#createFileSet(File, String, String)} using {@code /} as a path separator;
* the empty string means <em>no matches</em> (use {@link SelectorUtils#DEEP_TREE_MATCH} if you want to match everything except some excludes)
* @param excludes optional excludes in similar format to {@code includes}
* @param useDefaultExcludes as per {@link AbstractFileSet#setDefaultexcludes}
* @return a list of {@code /}-separated relative names of children (files directly inside or in subdirectories)
* @throws IOException if this is not a directory, or listing was not possible for some other reason
* @since FIXME
*/
public abstract @Nonnull String[] list(String glob) throws IOException;
@Restricted(Beta.class)
public @Nonnull Collection<String> list(@Nonnull String includes, @CheckForNull String excludes, boolean useDefaultExcludes) throws IOException {
Collection<String> r = run(new CollectFiles(this));
List<TokenizedPattern> includePatterns = patterns(includes);
List<TokenizedPattern> excludePatterns = patterns(excludes);
if (useDefaultExcludes) {
for (String patt : DirectoryScanner.getDefaultExcludes()) {
excludePatterns.add(new TokenizedPattern(patt.replace('/', File.separatorChar)));
}
}
return r.stream().filter(p -> {
TokenizedPath path = new TokenizedPath(p.replace('/', File.separatorChar));
return includePatterns.stream().anyMatch(patt -> patt.matchPath(path, true)) && !excludePatterns.stream().anyMatch(patt -> patt.matchPath(path, true));
}).collect(Collectors.toSet());
}
private static final class CollectFiles extends MasterToSlaveCallable<Collection<String>, IOException> {
private static final long serialVersionUID = 1;
private final VirtualFile root;
CollectFiles(VirtualFile root) {
this.root = root;
}
@Override
public Collection<String> call() throws IOException {
List<String> r = new ArrayList<>();
collectFiles(root, r, "");
return r;
}
private static void collectFiles(VirtualFile d, Collection<String> names, String prefix) throws IOException {
for (VirtualFile child : d.list()) {
if (child.isFile()) {
names.add(prefix + child.getName());
} else if (child.isDirectory()) {
collectFiles(child, names, prefix + child.getName() + "/");
}
}
}
}
private List<TokenizedPattern> patterns(String patts) {
List<TokenizedPattern> r = new ArrayList<>();
if (patts != null) {
for (String patt : patts.split(",")) {
if (patt.endsWith("/")) {
patt += SelectorUtils.DEEP_TREE_MATCH;
}
r.add(new TokenizedPattern(patt.replace('/', File.separatorChar)));
}
}
return r;
}
/**
* Obtains a child file.
......@@ -148,6 +261,18 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
*/
public abstract long lastModified() throws IOException;
/**
* Gets the file’s Unix mode, if meaningful.
* If the file is symlink (see {@link #readLink}), the mode is that of the link target, not the link itself.
* @return for example, 0644 ~ {@code rw-r--r--}; -1 by default, meaning unknown or inapplicable
* @throws IOException if checking the mode failed
* @since FIXME
*/
@Restricted(Beta.class)
public int mode() throws IOException {
return -1;
}
/**
* Checks whether this file can be read.
* @return true normally
......@@ -208,6 +333,26 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
return callable.call();
}
/**
* Optionally obtains a URL which may be used to retrieve file contents from any process on any node.
* For example, given cloud storage this might produce a permalink to the file.
* <p>This is only meaningful for {@link #isFile}:
* no ZIP etc. archiving protocol is defined to allow bulk access to directory trees.
* <p>Any necessary authentication must be encoded somehow into the URL itself;
* do not include any tokens or other authentication which might allow access to unrelated files
* (for example {@link ArtifactManager} builds from a different job).
* Authentication should be limited to download, not upload or any other modifications.
* <p>The URL might be valid for only a limited amount of time or even only a single use;
* this method should be called anew every time an external URL is required.
* @return an externally usable URL like {@code https://gist.githubusercontent.com/ACCT/GISTID/raw/COMMITHASH/FILE}, or null if there is no such support
* @since FIXME
* @see #toURI
*/
@Restricted(Beta.class)
public @CheckForNull URL toExternalURL() throws IOException {
return null;
}
/**
* Creates a virtual file wrapper for a local file.
* @param f a disk file (need not exist)
......@@ -250,6 +395,12 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
}
return f.exists();
}
@Override public String readLink() throws IOException {
if (isIllegalSymlink()) {
return null; // best to just ignore link -> ../whatever
}
return Util.resolveSymlink(f);
}
@Override public VirtualFile[] list() throws IOException {
if (isIllegalSymlink()) {
return new VirtualFile[0];
......@@ -264,11 +415,12 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
}
return vfs;
}
@Override public String[] list(String glob) throws IOException {
@Override
public Collection<String> list(String includes, String excludes, boolean useDefaultExcludes) throws IOException {
if (isIllegalSymlink()) {
return new String[0];
return Collections.emptySet();
}
return new Scanner(glob).invoke(f, null);
return new Scanner(includes, excludes, useDefaultExcludes).invoke(f, null);
}
@Override public VirtualFile child(String name) {
return new FileVF(new File(f, name), root);
......@@ -279,6 +431,12 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
}
return f.length();
}
@Override public int mode() throws IOException {
if (isIllegalSymlink()) {
return -1;
}
return IOUtils.mode(f);
}
@Override public long lastModified() throws IOException {
if (isIllegalSymlink()) {
return 0;
......@@ -348,7 +506,7 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
try {
return f.isDirectory();
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
@Override public boolean isFile() throws IOException {
......@@ -359,7 +517,14 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
try {
return f.exists();
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
@Override public String readLink() throws IOException {
try {
return f.readLink();
} catch (InterruptedException x) {
throw new IOException(x);
}
}
@Override public VirtualFile[] list() throws IOException {
......@@ -371,14 +536,14 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
}
return vfs;
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
@Override public String[] list(String glob) throws IOException {
@Override public Collection<String> list(String includes, String excludes, boolean useDefaultExcludes) throws IOException {
try {
return f.act(new Scanner(glob));
return f.act(new Scanner(includes, excludes, useDefaultExcludes));
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
@Override public VirtualFile child(String name) {
......@@ -388,52 +553,65 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
try {
return f.length();
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
@Override public int mode() throws IOException {
try {
return f.mode();
} catch (InterruptedException | PosixException x) {
throw new IOException(x);
}
}
@Override public long lastModified() throws IOException {
try {
return f.lastModified();
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
@Override public boolean canRead() throws IOException {
try {
return f.act(new Readable());
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
@Override public InputStream open() throws IOException {
try {
return f.read();
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
@Override public <V> V run(Callable<V,IOException> callable) throws IOException {
try {
return f.act(callable);
} catch (InterruptedException x) {
throw (IOException) new IOException(x.toString()).initCause(x);
throw new IOException(x);
}
}
}
private static final class Scanner extends MasterToSlaveFileCallable<String[]> {
private final String glob;
Scanner(String glob) {
this.glob = glob;
private static final class Scanner extends MasterToSlaveFileCallable<List<String>> {
private final String includes, excludes;
private final boolean useDefaultExcludes;
Scanner(String includes, String excludes, boolean useDefaultExcludes) {
this.includes = includes;
this.excludes = excludes;
this.useDefaultExcludes = useDefaultExcludes;
}
@Override public String[] invoke(File f, VirtualChannel channel) throws IOException {
@Override public List<String> invoke(File f, VirtualChannel channel) throws IOException {
if (includes.isEmpty()) { // see Glob class Javadoc, and list(String, String, boolean) note
return Collections.emptyList();
}
final List<String> paths = new ArrayList<String>();
new DirScanner.Glob(glob, null).scan(f, new FileVisitor() {
new DirScanner.Glob(includes, excludes, useDefaultExcludes).scan(f, new FileVisitor() {
@Override
public void visit(File f, String relativePath) throws IOException {
paths.add(relativePath);
paths.add(relativePath.replace('\\', '/'));
}
});
return paths.toArray(new String[paths.size()]);
return paths;
}
}
......
......@@ -24,14 +24,24 @@
package jenkins.util;
import com.google.common.collect.ImmutableSet;
import hudson.FilePath;
import hudson.Functions;
import hudson.Util;
import hudson.model.TaskListener;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.NoSuchFileException;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.NullInputStream;
import org.junit.Test;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeFalse;
......@@ -69,4 +79,120 @@ public class VirtualFileTest {
}
}
@Issue("JENKINS-26810")
@Test public void mode() throws Exception {
File f = tmp.newFile();
VirtualFile vf = VirtualFile.forFile(f);
FilePath fp = new FilePath(f);
VirtualFile vfp = VirtualFile.forFilePath(fp);
assertEquals(modeString(hudson.util.IOUtils.mode(f)), modeString(vf.mode()));
assertEquals(modeString(hudson.util.IOUtils.mode(f)), modeString(vfp.mode()));
fp.chmod(0755); // no-op on Windows, but harmless
assertEquals(modeString(hudson.util.IOUtils.mode(f)), modeString(vf.mode()));
assertEquals(modeString(hudson.util.IOUtils.mode(f)), modeString(vfp.mode()));
}
private static String modeString(int mode) throws IOException {
return mode == -1 ? "N/A" : PosixFilePermissions.toString(Util.modeToPermissions(mode));
}
@Issue("JENKINS-26810")
@Test public void list() throws Exception {
File root = tmp.getRoot();
FilePath rootF = new FilePath(root);
Set<String> paths = ImmutableSet.of("top.txt", "sub/mid.txt", "sub/subsub/lowest.txt", ".hg/config.txt", "very/deep/path/here");
for (String path : paths) {
rootF.child(path).write("", null);
}
for (VirtualFile vf : new VirtualFile[] {VirtualFile.forFile(root), VirtualFile.forFilePath(rootF), new Ram(paths.stream().map(p -> "/" + p).collect(Collectors.toSet()), "")}) {
System.err.println("testing " + vf.getClass().getName());
assertEquals("[.hg/config.txt, sub/mid.txt, sub/subsub/lowest.txt, top.txt]", new TreeSet<>(vf.list("**/*.txt", null, false)).toString());
assertEquals("[sub/mid.txt, sub/subsub/lowest.txt, top.txt]", new TreeSet<>(vf.list("**/*.txt", null, true)).toString());
assertEquals("[.hg/config.txt, sub/mid.txt, sub/subsub/lowest.txt, top.txt, very/deep/path/here]", new TreeSet<>(vf.list("**", null, false)).toString());
assertEquals("[]", new TreeSet<>(vf.list("", null, false)).toString());
assertEquals("[sub/mid.txt, sub/subsub/lowest.txt]", new TreeSet<>(vf.list("sub/", null, false)).toString());
assertEquals("[sub/mid.txt]", new TreeSet<>(vf.list("sub/", "sub/subsub/", false)).toString());
assertEquals("[sub/mid.txt]", new TreeSet<>(vf.list("sub/", "sub/subsub/**", false)).toString());
assertEquals("[sub/mid.txt]", new TreeSet<>(vf.list("sub/", "**/subsub/", false)).toString());
assertEquals("[.hg/config.txt, sub/mid.txt]", new TreeSet<>(vf.list("**/mid*,**/conf*", null, false)).toString());
assertEquals("[sub/mid.txt, sub/subsub/lowest.txt]", new TreeSet<>(vf.list("sub/", "**/notthere/", false)).toString());
assertEquals("[top.txt]", new TreeSet<>(vf.list("*.txt", null, false)).toString());
assertEquals("[sub/subsub/lowest.txt, top.txt, very/deep/path/here]", new TreeSet<>(vf.list("**", "**/mid*,**/conf*", false)).toString());
}
}
/** Roughly analogous to {@code org.jenkinsci.plugins.compress_artifacts.ZipStorage}. */
private static final class Ram extends VirtualFile {
private final Set<String> paths; // e.g., [/very/deep/path/here]
private final String path; // e.g., empty string or /very or /very/deep/path/here
Ram(Set<String> paths, String path) {
this.paths = paths;
this.path = path;
}
@Override
public String getName() {
return path.replaceFirst(".*/", "");
}
@Override
public URI toURI() {
return URI.create("ram:" + path);
}
@Override
public VirtualFile getParent() {
return new Ram(paths, path.replaceFirst("/[^/]+$", ""));
}
@Override
public boolean isDirectory() throws IOException {
return paths.stream().anyMatch(p -> p.startsWith(path + "/"));
}
@Override
public boolean isFile() throws IOException {
return paths.contains(path);
}
@Override
public boolean exists() throws IOException {
return isFile() || isDirectory();
}
@Override
public VirtualFile[] list() throws IOException {
return paths.stream().filter(p -> p.startsWith(path + "/")).map(p -> new Ram(paths, p.replaceFirst("(\\Q" + path + "\\E/[^/]+)/.+", "$1"))).toArray(VirtualFile[]::new);
}
@Override
public VirtualFile child(String name) {
return new Ram(paths, path + "/" + name);
}
@Override
public long length() throws IOException {
return 0;
}
@Override
public long lastModified() throws IOException {
return 0;
}
@Override
public boolean canRead() throws IOException {
return isFile();
}
@Override
public InputStream open() throws IOException {
return new NullInputStream(0);
}
}
@Issue("JENKINS-26810")
@Test public void readLink() throws Exception {
assumeFalse("Symlinks do not work well on Windows", Functions.isWindows());
File root = tmp.getRoot();
FilePath rootF = new FilePath(root);
rootF.child("plain").write("", null);
rootF.child("link").symlinkTo("physical", TaskListener.NULL);
for (VirtualFile vf : new VirtualFile[] {VirtualFile.forFile(root), VirtualFile.forFilePath(rootF)}) {
assertNull(vf.readLink());
assertNull(vf.child("plain").readLink());
VirtualFile link = vf.child("link");
assertEquals("physical", link.readLink());
assertFalse(link.isFile());
assertFalse(link.isDirectory());
// not checking .exists() for now
}
}
}
......@@ -93,7 +93,7 @@ THE SOFTWARE.
<matrix-project.version>1.4.1</matrix-project.version>
<sorcerer.version>0.11</sorcerer.version>
<animal.sniffer.skip>${skipTests}</animal.sniffer.skip>
<access-modifier.version>1.13</access-modifier.version>
<access-modifier.version>1.14</access-modifier.version>
<access-modifier-annotation.version>${access-modifier.version}</access-modifier-annotation.version> <!-- differing only where needed for timestamped snapshots -->
<access-modifier-checker.version>${access-modifier.version}</access-modifier-checker.version>
......
......@@ -23,6 +23,7 @@
*/
package hudson.model;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
......@@ -51,8 +52,28 @@ import org.jvnet.hudson.test.TestBuilder;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.UnexpectedPage;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import hudson.ExtensionList;
import hudson.Util;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import jenkins.model.ArtifactManager;
import jenkins.model.ArtifactManagerConfiguration;
import jenkins.model.ArtifactManagerFactory;
import jenkins.model.ArtifactManagerFactoryDescriptor;
import jenkins.model.Jenkins;
import jenkins.util.VirtualFile;
import org.apache.commons.io.IOUtils;
import org.jvnet.hudson.test.TestExtension;
import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;
/**
* @author Kohsuke Kawaguchi
......@@ -212,4 +233,200 @@ public class DirectoryBrowserSupportTest {
return file;
}
@Issue("JENKINS-49635")
@Test
public void externalURLDownload() throws Exception {
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(new ExternalArtifactManagerFactory());
FreeStyleProject p = j.createFreeStyleProject();
p.setScm(new SingleFileSCM("f", "Hello world!"));
p.getPublishersList().add(new ArtifactArchiver("f"));
j.buildAndAssertSuccess(p);
HtmlPage page = j.createWebClient().goTo("job/" + p.getName() + "/lastSuccessfulBuild/artifact/");
try {
Page download = page.getAnchorByText("f").click();
assertEquals("Hello world!", download.getWebResponse().getContentAsString());
} catch (FailingHttpStatusCodeException x) {
IOUtils.copy(x.getResponse().getContentAsStream(), System.err);
throw x;
}
}
/** Simulation of a storage service with URLs unrelated to {@link Run#doArtifact}. */
@TestExtension("externalURLDownload")
public static final class ContentAddressableStore implements UnprotectedRootAction {
final List<byte[]> files = new ArrayList<>();
@Override
public String getUrlName() {
return "files";
}
@Override
public String getIconFileName() {
return null;
}
@Override
public String getDisplayName() {
return null;
}
public void doDynamic(StaplerRequest req, StaplerResponse rsp) throws Exception {
String hash = req.getRestOfPath().substring(1);
for (byte[] file : files) {
if (Util.getDigestOf(new ByteArrayInputStream(file)).equals(hash)) {
rsp.setContentType("application/octet-stream");
rsp.getOutputStream().write(file);
return;
}
}
rsp.sendError(404);
}
}
public static final class ExternalArtifactManagerFactory extends ArtifactManagerFactory {
@Override
public ArtifactManager managerFor(Run<?, ?> build) {
return new ExternalArtifactManager();
}
@TestExtension("externalURLDownload")
public static final class DescriptorImpl extends ArtifactManagerFactoryDescriptor {}
}
private static final class ExternalArtifactManager extends ArtifactManager {
String hash;
@Override
public void archive(FilePath workspace, Launcher launcher, BuildListener listener, Map<String, String> artifacts) throws IOException, InterruptedException {
assertEquals(1, artifacts.size());
Map.Entry<String, String> entry = artifacts.entrySet().iterator().next();
assertEquals("f", entry.getKey());
try (InputStream is = workspace.child(entry.getValue()).read()) {
byte[] data = IOUtils.toByteArray(is);
ExtensionList.lookupSingleton(ContentAddressableStore.class).files.add(data);
hash = Util.getDigestOf(new ByteArrayInputStream(data));
}
}
@Override
public VirtualFile root() {
final VirtualFile file = new VirtualFile() { // the file inside the root
@Override
public String getName() {
return "f";
}
@Override
public URI toURI() {
return URI.create("root:f");
}
@Override
public VirtualFile getParent() {
return root();
}
@Override
public boolean isDirectory() throws IOException {
return false;
}
@Override
public boolean isFile() throws IOException {
return true;
}
@Override
public boolean exists() throws IOException {
return true;
}
@Override
public VirtualFile[] list() throws IOException {
return new VirtualFile[0];
}
@Override
public Collection<String> list(String includes, String excludes, boolean useDefaultExcludes) throws IOException {
return Collections.emptySet();
}
@Override
public VirtualFile child(String name) {
throw new UnsupportedOperationException();
}
@Override
public long length() throws IOException {
return 0;
}
@Override
public long lastModified() throws IOException {
return 0;
}
@Override
public boolean canRead() throws IOException {
return true;
}
@Override
public InputStream open() throws IOException {
throw new FileNotFoundException("expect to be opened via URL only");
}
@Override
public URL toExternalURL() throws IOException {
return new URL(Jenkins.get().getRootUrl() + "files/" + hash);
}
};
return new VirtualFile() { // the root
@Override
public String getName() {
return "";
}
@Override
public URI toURI() {
return URI.create("root:");
}
@Override
public VirtualFile getParent() {
return this;
}
@Override
public boolean isDirectory() throws IOException {
return true;
}
@Override
public boolean isFile() throws IOException {
return false;
}
@Override
public boolean exists() throws IOException {
return true;
}
@Override
public VirtualFile[] list() throws IOException {
return new VirtualFile[] {file};
}
@Override
public Collection<String> list(String includes, String excludes, boolean useDefaultExcludes) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public VirtualFile child(String name) {
if (name.equals("f")) {
return file;
} else if (name.isEmpty()) {
return this;
} else {
throw new UnsupportedOperationException("trying to call child on " + name);
}
}
@Override
public long length() throws IOException {
return 0;
}
@Override
public long lastModified() throws IOException {
return 0;
}
@Override
public boolean canRead() throws IOException {
return true;
}
@Override
public InputStream open() throws IOException {
throw new FileNotFoundException();
}
};
}
@Override
public void onLoad(Run<?, ?> build) {}
@Override
public boolean delete() throws IOException, InterruptedException {
return false;
}
}
}
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册