[FIXED JENKINS-19081] Offer the option of downloading metadata directly from the server.

(cherry picked from commit 1ac77750)

Conflicts:
	core/src/main/java/hudson/model/DownloadService.java
	core/src/main/java/hudson/model/UpdateSite.java

core/src/main/java/hudson/PluginManager.java

@@ -108,7 +108,12 @@ import org.xml.sax.SAXException;
 import org.xml.sax.helpers.DefaultHandler;
 
 import static hudson.init.InitMilestone.*;
+import hudson.model.DownloadService;
+import hudson.util.FormValidation;
 import static java.util.logging.Level.WARNING;
+import jenkins.security.DownloadSettings;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
 
 /**
  * Manages {@link PluginWrapper}s.
@@ -775,6 +780,24 @@ public abstract class PluginManager extends AbstractModelObject implements OnMas
         }
     }
 
+    @Restricted(NoExternalUse.class)
+    @RequirePOST public HttpResponse doCheckUpdatesServer() throws IOException {
+        for (UpdateSite site : Jenkins.getInstance().getUpdateCenter().getSites()) {
+            FormValidation v = site.updateDirectlyNow(DownloadSettings.get().isCheckSignature());
+            if (v.kind != FormValidation.Kind.OK) {
+                // TODO crude but enough for now
+                return v;
+            }
+        }
+        for (DownloadService.Downloadable d : DownloadService.Downloadable.all()) {
+            FormValidation v = d.updateNow();
+            if (v.kind != FormValidation.Kind.OK) {
+                return v;
+            }
+        }
+        return HttpResponses.forwardToPreviousPage();
+    }
+
     protected String identifyPluginShortName(File t) {
         try {
             JarFile j = new JarFile(t);

core/src/main/java/hudson/model/DownloadService.java

@@ -26,20 +26,25 @@ package hudson.model;
 import hudson.Extension;
 import hudson.ExtensionList;
 import hudson.ExtensionPoint;
+import hudson.ProxyConfiguration;
 import hudson.util.FormValidation;
 import hudson.util.FormValidation.Kind;
 import hudson.util.QuotedStringTokenizer;
 import hudson.util.TextFile;
-import jenkins.model.Jenkins;
-import jenkins.util.JSONSignatureValidator;
-import net.sf.json.JSONException;
-import org.kohsuke.stapler.Stapler;
-
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLEncoder;
 import java.util.logging.Logger;
-
+import jenkins.security.DownloadSettings;
+import jenkins.model.Jenkins;
+import jenkins.util.JSONSignatureValidator;
+import net.sf.json.JSONException;
 import net.sf.json.JSONObject;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+import org.kohsuke.stapler.Stapler;
 import org.kohsuke.stapler.StaplerRequest;
 import org.kohsuke.stapler.StaplerResponse;
 
@@ -62,6 +67,9 @@ public class DownloadService extends PageDecorator {
      * Builds up an HTML fragment that starts all the download jobs.
      */
     public String generateFragment() {
+        if (!DownloadSettings.get().isUseBrowser()) {
+            return "";
+        }
     	if (neverUpdate) return "";
         if (doesNotSupportPostMessage())  return "";
 
@@ -135,6 +143,34 @@ public class DownloadService extends PageDecorator {
         return null;
     }
 
+    /**
+     * Loads JSON from a JSONP URL.
+     * Metadata for downloadables and update centers is offered in two formats, both designed for download from the browser (predating {@link DownloadSettings}):
+     * HTML using {@code postMessage} for newer browsers, and JSONP as a fallback.
+     * Confusingly, the JSONP files are given the {@code *.json} file extension, when they are really JavaScript and should be {@code *.js}.
+     * This method extracts the JSON from a JSONP URL, since that is what we actually want when we download from the server.
+     * (Currently the true JSON is not published separately, and extracting from the {@code *.json.html} is more work.)
+     * @param src a URL to a JSONP file (typically including {@code id} and {@code version} query parameters)
+     * @return the embedded JSON text
+     * @throws IOException if either downloading or processing failed
+     */
+    @Restricted(NoExternalUse.class)
+    public static String loadJSON(URL src) throws IOException {
+        InputStream is = ProxyConfiguration.open(src).getInputStream();
+        try {
+            String jsonp = IOUtils.toString(is, "UTF-8");
+            int start = jsonp.indexOf('{');
+            int end = jsonp.lastIndexOf('}');
+            if (start >= 0 && end > start) {
+                return jsonp.substring(start, end + 1);
+            } else {
+                throw new IOException("Could not find JSON in " + src);
+            }
+        } finally {
+            is.close();
+        }
+    }
+
     /**
      * Represents a periodically updated JSON data file obtained from a remote URL.
      *
@@ -248,17 +284,28 @@ public class DownloadService extends PageDecorator {
          * This is where the browser sends us the data. 
          */
         public void doPostBack(StaplerRequest req, StaplerResponse rsp) throws IOException {
+            if (!DownloadSettings.get().isUseBrowser()) {
+                throw new IOException("not allowed");
+            }
             long dataTimestamp = System.currentTimeMillis();
             due = dataTimestamp+getInterval();  // success or fail, don't try too often
 
             String json = IOUtils.toString(req.getInputStream(),"UTF-8");
+            FormValidation e = load(json, dataTimestamp);
+            if (e.kind != Kind.OK) {
+                LOGGER.severe(e.renderHtml());
+                throw e;
+            }
+            rsp.setContentType("text/plain");  // So browser won't try to parse response
+        }
+
+        private FormValidation load(String json, long dataTimestamp) throws IOException {
             JSONObject o = JSONObject.fromObject(json);
 
-            if (signatureCheck) {
+            if (DownloadSettings.get().isCheckSignature()) {
                 FormValidation e = new JSONSignatureValidator("downloadable '"+id+"'").verifySignature(o);
                 if (e.kind!= Kind.OK) {
-                    LOGGER.severe(e.renderHtml());
-                    throw e;
+                    return e;
                 }
             }
 
@@ -266,8 +313,12 @@ public class DownloadService extends PageDecorator {
             df.write(json);
             df.file.setLastModified(dataTimestamp);
             LOGGER.info("Obtained the updated data file for "+id);
+            return FormValidation.ok();
+        }
 
-            rsp.setContentType("text/plain");  // So browser won't try to parse response
+        @Restricted(NoExternalUse.class)
+        public FormValidation updateNow() throws IOException {
+            return load(loadJSON(new URL(getUrl() + "?id=" + URLEncoder.encode(getId(), "UTF-8") + "&version=" + URLEncoder.encode(Jenkins.VERSION, "UTF-8"))), System.currentTimeMillis());
         }
 
         /**
@@ -295,9 +346,7 @@ public class DownloadService extends PageDecorator {
 
     public static boolean neverUpdate = Boolean.getBoolean(DownloadService.class.getName()+".never");
 
-    /**
-     * Off by default until we know this is reasonably working.
-     */
+    /** @deprecated Use {@link DownloadSettings#setIgnoreSignature} instead. */
     public static boolean signatureCheck = !Boolean.getBoolean(DownloadService.class.getName()+".noSignatureCheck");
 }
 

core/src/main/java/hudson/model/UpdateCenter.java

@@ -88,6 +88,7 @@ import java.util.jar.JarFile;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.annotation.CheckForNull;
+import jenkins.security.DownloadSettings;
 import org.acegisecurity.context.SecurityContextHolder;
 import org.kohsuke.accmod.Restricted;
 import org.kohsuke.accmod.restrictions.NoExternalUse;
@@ -630,7 +631,7 @@ public class UpdateCenter extends AbstractModelObject implements Saveable, OnMas
     public List<FormValidation> updateAllSites() throws InterruptedException, ExecutionException {
         List <Future<FormValidation>> futures = new ArrayList<Future<FormValidation>>();
         for (UpdateSite site : getSites()) {
-            Future<FormValidation> future = site.updateDirectly(true);
+            Future<FormValidation> future = site.updateDirectly(DownloadSettings.get().isCheckSignature());
             if (future != null) {
                 futures.add(future);
             }

core/src/main/java/hudson/model/UpdateSite.java

@@ -27,32 +27,18 @@ package hudson.model;
 
 import hudson.PluginManager;
 import hudson.PluginWrapper;
-import hudson.ProxyConfiguration;
 import hudson.lifecycle.Lifecycle;
 import hudson.model.UpdateCenter.UpdateCenterJob;
 import hudson.util.FormValidation;
 import hudson.util.FormValidation.Kind;
 import hudson.util.HttpResponses;
 import hudson.util.TextFile;
+import static hudson.util.TimeUnit2.*;
 import hudson.util.VersionNumber;
-import jenkins.model.Jenkins;
-import jenkins.util.JSONSignatureValidator;
-import net.sf.json.JSONException;
-import net.sf.json.JSONObject;
-import org.apache.commons.lang.StringUtils;
-import org.kohsuke.stapler.DataBoundConstructor;
-import org.kohsuke.stapler.HttpResponse;
-import org.kohsuke.stapler.StaplerRequest;
-import org.kohsuke.stapler.export.Exported;
-import org.kohsuke.stapler.export.ExportedBean;
-import org.kohsuke.stapler.interceptor.RequirePOST;
-
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.URI;
 import java.net.URL;
-import java.net.URLConnection;
 import java.net.URLEncoder;
 import java.security.GeneralSecurityException;
 import java.util.ArrayList;
@@ -66,10 +52,23 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
-import static hudson.util.TimeUnit2.*;
 import org.apache.commons.io.IOUtils;
-
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+import jenkins.model.Jenkins;
+import jenkins.security.DownloadSettings;
+import jenkins.util.JSONSignatureValidator;
+import net.sf.json.JSONException;
+import net.sf.json.JSONObject;
+import org.apache.commons.lang.StringUtils;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+import org.kohsuke.stapler.DataBoundConstructor;
+import org.kohsuke.stapler.HttpResponse;
+import org.kohsuke.stapler.StaplerRequest;
+import org.kohsuke.stapler.export.Exported;
+import org.kohsuke.stapler.export.ExportedBean;
+import org.kohsuke.stapler.interceptor.RequirePOST;
 
 /**
  * Source of the update center information, like "http://jenkins-ci.org/update-center.json"
@@ -154,42 +153,30 @@ public class UpdateSite {
      * @return null if no updates are necessary, or the future result
      * @since 1.502
      */
-    public Future<FormValidation> updateDirectly(final boolean signatureCheck) {
+    public @CheckForNull Future<FormValidation> updateDirectly(final boolean signatureCheck) {
         if (! getDataFile().exists() || isDue()) {
             return Jenkins.getInstance().getUpdateCenter().updateService.submit(new Callable<FormValidation>() {
-                
-                public FormValidation call() throws Exception {
-                    URL src = new URL(getUrl() + "?id=" + URLEncoder.encode(getId(),"UTF-8") 
-                            + "&version="+URLEncoder.encode(Jenkins.VERSION, "UTF-8"));
-                    URLConnection conn = ProxyConfiguration.open(src);
-                    InputStream is = conn.getInputStream();
-                    try {
-                        String uncleanJson = IOUtils.toString(is,"UTF-8");
-                        int jsonStart = uncleanJson.indexOf("{\"");
-                        if (jsonStart >= 0) {
-                            uncleanJson = uncleanJson.substring(jsonStart);
-                            int end = uncleanJson.lastIndexOf('}');
-                            if (end>0)
-                                uncleanJson = uncleanJson.substring(0,end+1);
-                            return updateData(uncleanJson, signatureCheck);
-                        } else {
-                            throw new IOException("Could not find json in content of " +
-                            		"update center from url: "+src.toExternalForm());
-                        }
-                    } finally {
-                        if (is != null)
-                            is.close();
-                    }
+                @Override public FormValidation call() throws Exception {
+                    return updateDirectlyNow(signatureCheck);
                 }
             });
-        }
+        } else {
             return null;
+        }
+    }
+
+    @Restricted(NoExternalUse.class)
+    public @Nonnull FormValidation updateDirectlyNow(boolean signatureCheck) throws IOException {
+        return updateData(DownloadService.loadJSON(new URL(getUrl() + "?id=" + URLEncoder.encode(getId(), "UTF-8") + "&version=" + URLEncoder.encode(Jenkins.VERSION, "UTF-8"))), signatureCheck);
     }
     
     /**
      * This is the endpoint that receives the update center data file from the browser.
      */
     public FormValidation doPostBack(StaplerRequest req) throws IOException, GeneralSecurityException {
+        if (!DownloadSettings.get().isUseBrowser()) {
+            throw new IOException("not allowed");
+        }
         return updateData(IOUtils.toString(req.getInputStream(),"UTF-8"), true);
     }
 

core/src/main/java/hudson/util/FormValidation.java

@@ -239,9 +239,13 @@ public abstract class FormValidation extends IOException implements HttpResponse
             return ok();
         return new FormValidation(kind, message) {
             public String renderHtml() {
+                StaplerRequest req = Stapler.getCurrentRequest();
+                if (req == null) { // being called from some other context
+                    return message;
+                }
                 // 1x16 spacer needed for IE since it doesn't support min-height
                 return "<div class="+ kind.name().toLowerCase(Locale.ENGLISH) +"><img src='"+
-                        Stapler.getCurrentRequest().getContextPath()+ Jenkins.RESOURCE_PATH+"/images/none.gif' height=16 width=1>"+
+                        req.getContextPath()+ Jenkins.RESOURCE_PATH+"/images/none.gif' height=16 width=1>"+
                         message+"</div>";
             }
             @Override public String toString() {

core/src/main/java/jenkins/security/DownloadSettings.java

+/*
+ * The MIT License
+ *
+ * Copyright 2014 Jesse Glick.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package jenkins.security;
+
+import hudson.Extension;
+import hudson.model.AsyncPeriodicWork;
+import hudson.model.DownloadService;
+import hudson.model.TaskListener;
+import hudson.model.UpdateSite;
+import hudson.util.FormValidation;
+import java.io.IOException;
+import jenkins.model.GlobalConfiguration;
+import jenkins.model.GlobalConfigurationCategory;
+import jenkins.model.Jenkins;
+import net.sf.json.JSONObject;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+import org.kohsuke.stapler.HttpResponse;
+import org.kohsuke.stapler.QueryParameter;
+import org.kohsuke.stapler.StaplerRequest;
+
+/**
+ * Lets user configure how metadata files should be downloaded.
+ * @see UpdateSite
+ * @see DownloadService
+ */
+@Restricted(NoExternalUse.class) // no clear reason for this to be an API
+@Extension public final class DownloadSettings extends GlobalConfiguration {
+
+    public static DownloadSettings get() {
+        return Jenkins.getInstance().getInjector().getInstance(DownloadSettings.class);
+    }
+
+    private boolean useBrowser = true; // historical default, not necessarily recommended
+    @SuppressWarnings("deprecation")
+    private boolean checkSignature = DownloadService.signatureCheck;
+    
+    public DownloadSettings() {
+        load();
+    }
+
+    @Override public GlobalConfigurationCategory getCategory() {
+        return GlobalConfigurationCategory.get(GlobalConfigurationCategory.Security.class);
+    }
+
+    @Override public boolean configure(StaplerRequest req, JSONObject json) throws FormException {
+        req.bindJSON(this, json);
+        return true;
+    }
+
+    public boolean isUseBrowser() {
+        return useBrowser;
+    }
+
+    public void setUseBrowser(boolean useBrowser) {
+        this.useBrowser = useBrowser;
+        save();
+    }
+
+    public boolean isCheckSignature() {
+        return checkSignature;
+    }
+    
+    public void setCheckSignature(boolean checkSignature) {
+        this.checkSignature = checkSignature;
+        save();
+    }
+
+    public FormValidation doCheckCheckSignature(@QueryParameter boolean value, @QueryParameter boolean useBrowser) {
+        if (value) {
+            return FormValidation.ok();
+        } else if (useBrowser) {
+            return FormValidation.warningWithMarkup(Messages.DownloadSettings_disabling_signature_checks_for_in_browse());
+        } else {
+            return FormValidation.warningWithMarkup(Messages.DownloadSettings_disabling_signature_checks_is_not_recomm());
+        }
+    }
+
+    @Extension public static final class DailyCheck extends AsyncPeriodicWork {
+
+        public DailyCheck() {
+            super("Download metadata");
+        }
+
+        @Override public long getRecurrencePeriod() {
+            return DAY;
+        }
+
+        @Override protected void execute(TaskListener listener) throws IOException, InterruptedException {
+            if (get().isUseBrowser()) {
+                return;
+            }
+            HttpResponse rsp = Jenkins.getInstance().getPluginManager().doCheckUpdatesServer();
+            if (rsp instanceof FormValidation) {
+                listener.error(((FormValidation) rsp).renderHtml());
+            }
+        }
+
+    }
+
+}

core/src/main/resources/hudson/PluginManager/advanced.jelly

@@ -96,7 +96,8 @@ THE SOFTWARE.
       </tr>
     </table>
     <div align="right" style="margin-top:1em">
-      <form method="post" action="checkUpdates">
+      <j:invokeStatic var="ds" className="jenkins.security.DownloadSettings" method="get"/>
+      <form method="post" action="${ds.useBrowser ? 'checkUpdates' : 'checkUpdatesServer'}">
         ${%lastUpdated(app.updateCenter.lastUpdatedString)}
         <f:submit value="${%Check now}" />
       </form>

core/src/main/resources/hudson/model/UpdateCenter/PageDecoratorImpl/footer.jelly

@@ -31,6 +31,8 @@ THE SOFTWARE.
 -->
 <?jelly escape-by-default='true'?>
 <j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
+  <j:invokeStatic var="ds" className="jenkins.security.DownloadSettings" method="get"/>
+  <j:if test="${ds.useBrowser}">
   <j:forEach var="site" items="${app.updateCenter.sites}">
     <j:if test="${site.due or forcedUpdateCheck}">
       <script>
@@ -45,4 +47,5 @@ THE SOFTWARE.
       </script>
     </j:if>
   </j:forEach>
+  </j:if>
 </j:jelly>

core/src/main/resources/jenkins/security/DownloadSettings/config.groovy

+package jenkins.security.DownloadSettings
+
+def f = namespace(lib.FormTagLib)
+
+f.section(title: _("Download Preferences")) {
+    f.entry(title: _("Use Browser"), field: "useBrowser") {
+        f.checkbox()
+    }
+    f.entry(title: _("Check Signatures"), field: "checkSignature") {
+        f.checkbox()
+    }
+}

core/src/main/resources/jenkins/security/DownloadSettings/help-checkSignature.html

+<div>
+    Uncheck to skip the signature check on downloaded metadata.
+    This is generally dangerous and should only be used as an emergency measure when dealing with broken upstream metadata.
+</div>

core/src/main/resources/jenkins/security/DownloadSettings/help-useBrowser.html

+<div>
+    Check to force the user’s browser to download metadata rather than Jenkins itself doing it.
+    Actual file downloads (plugins, tools) will still happen from Jenkins itself,
+    but this can be used to at least <em>see</em> new metadata when Jenkins cannot access the Internet
+    (but your own browser can, perhaps using some special proxy that Jenkins is not configured to use).
+</div>

core/src/main/resources/jenkins/security/Messages.properties

@@ -22,4 +22,6 @@
 
 ApiTokenProperty.DisplayName=API Token
 ApiTokenProperty.ChangeToken.Success=<div>Updated</div>
-RekeySecretAdminMonitor.DisplayName=Re-keying
\ No newline at end of file
+DownloadSettings.disabling_signature_checks_for_in_browse=Disabling signature checks for in-browser downloads is <em>very dangerous</em> as unprivileged users could submit spoofed metadata!
+DownloadSettings.disabling_signature_checks_is_not_recomm=Disabling signature checks is not recommended except as a temporary measure when upstream metadata is broken, such as due to expired certificates.
+RekeySecretAdminMonitor.DisplayName=Re-keying

test/src/test/java/hudson/model/DownloadServiceTest.java

@@ -7,6 +7,7 @@ import org.jvnet.hudson.test.HudsonTestCase;
 import org.kohsuke.stapler.StaplerResponse;
 
 import java.io.IOException;
+import jenkins.security.DownloadSettings;
 
 /**
  * @author Kohsuke Kawaguchi
@@ -40,13 +41,13 @@ public class DownloadServiceTest extends HudsonTestCase {
         assertNull(job.getData());
 
         // and now it should work
-        DownloadService.signatureCheck = false;
+        DownloadSettings.get().setCheckSignature(false);
         try {
             createWebClient().goTo("/self/testPost");
             JSONObject d = job.getData();
             assertEquals(hashCode(),d.getInt("hello"));
         } finally {
-            DownloadService.signatureCheck = true;
+            DownloadSettings.get().setCheckSignature(true);
         }
 
         // TODO: test with a signature