Added an ability to collect hs_err_pid*.log files

changelog.html

@@ -64,6 +64,8 @@ Upcoming changes</a>
   <li class=rfe>
     Add Test button to check proxy connection
     (<a href="https://issues.jenkins-ci.org/browse/JENKINS-20191">issue 20191</a>)
+  <li class=rfe>
+    Collect and report JVM crash dump files to assist trouble-shooting
   <li class=rfe>
     Show different “up” link for jobs in folders.
     (<a href="https://issues.jenkins-ci.org/browse/JENKINS-20106">issue 20106</a>)

core/src/main/java/hudson/util/jna/Kernel32.java

@@ -72,5 +72,6 @@ public interface Kernel32 extends StdCallLibrary {
 
     int STILL_ACTIVE = 259;
 
+    int GetTempPathW(int nBuffer, Pointer lpBuffer);
     // DWORD == int
 }

core/src/main/java/hudson/util/jna/Kernel32Utils.java

@@ -27,6 +27,7 @@ import java.io.*;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import com.sun.jna.Memory;
 import com.sun.jna.Native;
 import com.sun.jna.Pointer;
 import com.sun.jna.ptr.IntByReference;
@@ -97,6 +98,15 @@ public class Kernel32Utils {
         return (file.exists() && (Kernel32.FILE_ATTRIBUTE_REPARSE_POINT & getWin32FileAttributes(file)) != 0);
     }
 
+    public static File getTempDir() {
+        Memory buf = new Memory(1024);
+        if (Kernel32.INSTANCE.GetTempPathW(1024,buf)!=0) {
+            return new File(buf.getString(0, true));
+        } else {
+            return null;
+        }
+    }
+
     /*package*/ static Kernel32 load() {
         try {
             return (Kernel32) Native.loadLibrary("kernel32", Kernel32.class);

core/src/main/java/jenkins/diagnosis/HsErrPidFile.java

+package jenkins.diagnosis;
+
+import hudson.Util;
+import hudson.util.HttpResponses;
+import jenkins.model.Jenkins;
+import org.kohsuke.stapler.HttpResponse;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * Serves hs_err_pid file.
+ *
+ * @author Kohsuke Kawaguchi
+ */
+public class HsErrPidFile {
+    private final HsErrPidList owner;
+    private final File file;
+
+    public HsErrPidFile(HsErrPidList owner, File file) {
+        this.owner = owner;
+        this.file = file;
+    }
+
+    public String getName() {
+        return file.getName();
+    }
+
+    public String getPath() {
+        return file.getPath();
+    }
+
+    public long getLastModified() {
+        return file.lastModified();
+    }
+
+    public Date getLastModifiedDate() {
+        return new Date(file.lastModified());
+    }
+
+    public String getTimeSpanString() {
+        return Util.getTimeSpanString(System.currentTimeMillis()-getLastModified());
+    }
+
+    public HttpResponse doDownload() throws IOException {
+        Jenkins.getInstance().checkPermission(Jenkins.ADMINISTER);
+        return HttpResponses.staticResource(file);
+    }
+
+    public HttpResponse doDelete() throws IOException {
+        Jenkins.getInstance().checkPermission(Jenkins.ADMINISTER);
+        file.delete();
+        owner.files.remove(this);
+        return HttpResponses.redirectTo("../..");
+    }
+}

core/src/main/java/jenkins/diagnosis/HsErrPidList.java

+package jenkins.diagnosis;
+
+import com.sun.akuma.JavaVMArguments;
+import hudson.Extension;
+import hudson.Functions;
+import hudson.Util;
+import hudson.model.AdministrativeMonitor;
+import hudson.util.IOUtils;
+import hudson.util.jna.Kernel32Utils;
+import jenkins.model.Jenkins;
+import org.apache.tools.ant.DirectoryScanner;
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.types.FileSet;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Finds crash dump reports and show them in the UI.
+ *
+ * @author Kohsuke Kawaguchi
+ */
+@Extension(optional=true)
+public class HsErrPidList extends AdministrativeMonitor {
+    /**
+     * hs_err_pid files that we think belong to us.
+     */
+    /*package*/ final List<HsErrPidFile> files = new ArrayList<HsErrPidFile>();
+
+    /**
+     * Used to keep a marker file memory-mapped, so that we can find hs_err_pid files that belong to us.
+     */
+    private MappedByteBuffer map;
+
+    public HsErrPidList() {
+        try {
+            FileChannel ch = new FileInputStream(getSecretKeyFile()).getChannel();
+            map = ch.map(MapMode.READ_ONLY,0,1);
+
+            scan("./hs_err_pid%p.log");
+            if (Functions.isWindows()) {
+                File dir = Kernel32Utils.getTempDir();
+                if (dir!=null) {
+                    scan(dir.getPath() + "\\hs_err_pid%p.log");
+                }
+            } else {
+                scan("/tmp/hs_err_pid%p.log");
+            }
+            // on different platforms, rules about the default locations are a lot more subtle.
+
+            // check our arguments in the very end since this might fail on some platforms
+            JavaVMArguments args = JavaVMArguments.current();
+            for (String a : args) {
+                // see http://www.oracle.com/technetwork/java/javase/felog-138657.html
+                if (a.startsWith(ERROR_FILE_OPTION)) {
+                    scan(a.substring(ERROR_FILE_OPTION.length()));
+                }
+            }
+        } catch (UnsupportedOperationException e) {
+            // ignore
+        } catch (Throwable e) {
+            LOGGER.log(Level.WARNING, "Failed to list up hs_err_pid files", e);
+        }
+    }
+
+    @Override
+    public String getDisplayName() {
+        return "JVM Crash Reports";
+    }
+
+    /**
+     * Expose files to the URL.
+     */
+    public List<HsErrPidFile> getFiles() {
+        return files;
+    }
+
+
+    private void scan(String pattern) {
+        LOGGER.fine("Scanning "+pattern+" for hs_err_pid files");
+
+        pattern = pattern.replace("%p","*").replace("%%","%");
+        File f = new File(pattern).getAbsoluteFile();
+        if (!pattern.contains("*"))
+            scanFile(f);
+        else {// GLOB
+            File commonParent = f;
+            while (commonParent!=null && commonParent.getPath().contains("*")) {
+                commonParent = commonParent.getParentFile();
+            }
+            if (commonParent==null) {
+                LOGGER.warning("Failed to process "+f);
+                return; // huh?
+            }
+
+            FileSet fs = Util.createFileSet(commonParent, f.getPath().substring(commonParent.getPath().length()+1), null);
+            DirectoryScanner ds = fs.getDirectoryScanner(new Project());
+            for (String child : ds.getIncludedFiles()) {
+                scanFile(new File(commonParent,child));
+            }
+        }
+    }
+
+    private void scanFile(File log) {
+        LOGGER.fine("Scanning "+log);
+
+        BufferedReader r=null;
+        try {
+            r = new BufferedReader(new FileReader(log));
+
+            if (!findHeader(r))
+                return;
+
+            // we should find a memory mapped file for secret.key
+            String secretKey = getSecretKeyFile().getAbsolutePath();
+
+
+            String line;
+            while ((line=r.readLine())!=null) {
+                if (line.contains(secretKey)) {
+                    files.add(new HsErrPidFile(this,log));
+                    return;
+                }
+            }
+        } catch (IOException e) {
+            // not a big enough deal.
+            LOGGER.log(Level.FINE, "Failed to parse hs_err_pid file: " + log, e);
+        } finally {
+            IOUtils.closeQuietly(r);
+        }
+    }
+
+    private File getSecretKeyFile() {
+        return new File(Jenkins.getInstance().getRootDir(),"secret.key");
+    }
+
+    private boolean findHeader(BufferedReader r) throws IOException {
+        for (int i=0; i<5; i++) {
+            String line = r.readLine();
+            if (line==null)
+                return false;
+            if (line.startsWith("# A fatal error has been detected by the Java Runtime Environment:"))
+                return true;
+        }
+        return false;
+    }
+
+    @Override
+    public boolean isActivated() {
+        return !files.isEmpty();
+    }
+
+    private static final String ERROR_FILE_OPTION = "-XX:ErrorFile=";
+    private static final Logger LOGGER = Logger.getLogger(HsErrPidList.class.getName());
+}

core/src/main/resources/jenkins/diagnosis/HsErrPidList/index.jelly

+<!--
+The MIT License
+
+Copyright (c) 2013-, CloudBees, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+-->
+
+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:l="/lib/layout" xmlns:f="/lib/form" xmlns:i="jelly:fmt" xmlns:st="jelly:stapler">
+  <l:layout title="${%Java VM Crash Reports}">
+    <st:include page="sidepanel" it="${app}"/>
+    <l:main-panel>
+      <h1>${%Java VM Crash Reports}</h1>
+
+      <p>${%blurb}</p>
+
+      <table class="bigtable sortable pane">
+        <tr>
+          <th>${%Name}</th>
+          <th>${%Date}</th>
+          <th/>
+        </tr>
+        <j:forEach var="f" items="${it.files}" varStatus="idx">
+          <tr>
+            <td style="padding-right:3em">
+              <img src="${imagesURL}/32x32/clipboard.png" style="margin-right:1em"/>
+              <a href="files/${idx.index}/download/${f.name}">
+                ${f.path}
+              </a>
+            </td>
+            <td data="${f.lastModified}">
+              <i:formatDate value="${f.lastModifiedDate}" type="both" dateStyle="medium" timeStyle="medium"/>
+              <st:nbsp />
+              (${%ago(f.timeSpanString)})
+            </td>
+            <td>
+              <form method="post" action="files/${idx.index}/${f.name}/delete" name="${idx}">
+                <f:submit value="${%Delete}"/>
+              </form>
+            </td>
+          </tr>
+        </j:forEach>
+      </table>
+    </l:main-panel>
+  </l:layout>
+</j:jelly>
\ No newline at end of file

core/src/main/resources/jenkins/diagnosis/HsErrPidList/index.properties

+blurb=The following JVM crash reports are found for this Jenkins instance. \
+  If you think this is a problem in Jenkins, please <a href="https://issues.jenkins-ci.org/">report it</a>. \
+  Jenkins relies on some heuristics to find these files. For more reliable discovery, please consider adding \
+  <tt>-XX:ErrorFile=/path/to/hs_err_pid%p.log</tt> as your JVM argument.
+ago={0} ago
\ No newline at end of file

core/src/main/resources/jenkins/diagnosis/HsErrPidList/message.jelly

+<!--
+The MIT License
+
+Copyright (c) 2013-, CloudBees, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+-->
+
+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
+  <div class="error">
+    ${%blurb(rootURL+'/'+it.url)}
+  </div>
+</j:jelly>
\ No newline at end of file

core/src/main/resources/jenkins/diagnosis/HsErrPidList/message.properties

+blurb=This Jenkins appears to have crashed. <a href="{0}">Please check the logs</a>.
\ No newline at end of file